Merge pull request #796 from netbox-community/develop

Version 2.0.0

.github/workflows/push.yml

@@ -13,11 +13,13 @@ jobs:
     runs-on: ubuntu-latest
     name: Checks syntax of our code
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         # Full git history is needed to get a proper list of changed files within `super-linter`
         fetch-depth: 0
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
     - name: Lint Code Base
       uses: github/super-linter@v4
       env:
@@ -46,14 +48,13 @@ jobs:
         - ./build.sh develop
         docker_from:
         - '' # use the default of the build script
-        - alpine:edge
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new NetBox Docker Images
     steps:
     - id: git-checkout
       name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - id: docker-build
       name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}'
       run: ${{ matrix.build_cmd }}

.github/workflows/release.yml

@@ -19,33 +19,21 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new NetBox Docker Images
+    env:
+      GH_ACTION: enable
     steps:
     -
       name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     -
       name: Get Version of NetBox Docker
       run: |
         echo "::set-output name=version::$(cat VERSION)"
       shell: bash
-    -
-      name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
-    -
-      id: buildx-setup
-      name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
-      with:
-        install: true
-    -
-      name: Available platforms
-      run: echo ${{ steps.buildx-setup.outputs.platforms }}
     -
       id: docker-build
       name: Build the image with '${{ matrix.build_cmd }}'
       run: ${{ matrix.build_cmd }}
-      env:
-        GH_ACTION: enable
     -
       name: Test the image
       run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh
@@ -54,7 +42,7 @@ jobs:
     # docker.io
     -
       name: Login to docker.io
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry: docker.io
         username: ${{ secrets.dockerhub_username }}
@@ -68,7 +56,7 @@ jobs:
     # quay.io
     -
       name: Login to Quay.io
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry: quay.io
         username: ${{ secrets.quayio_username }}
@@ -79,13 +67,12 @@ jobs:
       run: ${{ matrix.build_cmd }} --push
       env:
         DOCKER_REGISTRY: quay.io
-        GH_ACTION: enable
       if: steps.docker-build.outputs.skipped != 'true'
 
     # ghcr.io
     -
       name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
@@ -97,5 +84,4 @@ jobs:
       env:
         DOCKER_REGISTRY: ghcr.io
         DOCKER_ORG: netbox-community
-        GH_ACTION: enable
       if: steps.docker-build.outputs.skipped != 'true'

.hadolint.yaml

@@ -1,3 +1,4 @@
 ignored:
 - DL3006
-- DL3018
+- DL3008
+- DL3003

Dockerfile

@@ -1,45 +1,27 @@
 ARG FROM
 FROM ${FROM} as builder
 
-RUN apk add --no-cache \
-      bash \
-      build-base \
-      cargo \
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update -qq \
+    && apt-get upgrade \
+      --yes -qq --no-install-recommends \
+    && apt-get install \
+      --yes -qq --no-install-recommends \
+      build-essential \
       ca-certificates \
-      cmake \
-      cyrus-sasl-dev \
-      git \
-      graphviz \
-      jpeg-dev \
-      libevent-dev \
-      libffi-dev \
-      libxslt-dev \
-      make \
-      musl-dev \
-      openldap-dev \
-      postgresql-dev \
-      py3-pip \
+      libldap-dev \
+      libpq-dev \
+      libsasl2-dev \
+      libssl-dev \
       python3-dev \
-  && python3 -m venv /opt/netbox/venv \
-  && /opt/netbox/venv/bin/python3 -m pip install --upgrade \
+      python3-pip \
+      python3-venv \
+    && python3 -m venv /opt/netbox/venv \
+    && /opt/netbox/venv/bin/python3 -m pip install --upgrade \
       pip \
       setuptools \
       wheel
 
-# Build libcrc32c for google-crc32c python module
-RUN git clone https://github.com/google/crc32c \
-    && cd crc32c \
-    && git submodule update --init --recursive \
-    && mkdir build \
-    && cd build \
-    && cmake \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DCRC32C_BUILD_TESTS=no \
-        -DCRC32C_BUILD_BENCHMARKS=no \
-        -DBUILD_SHARED_LIBS=yes \
-        .. \
-    && make all install
-
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
 RUN /opt/netbox/venv/bin/pip install \
@@ -53,35 +35,37 @@ RUN /opt/netbox/venv/bin/pip install \
 ARG FROM
 FROM ${FROM} as main
 
-RUN apk add --no-cache \
-      bash \
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update -qq \
+    && apt-get upgrade \
+      --yes -qq --no-install-recommends \
+    && apt-get install \
+      --yes -qq --no-install-recommends \
       ca-certificates \
       curl \
-      graphviz \
-      libevent \
-      libffi \
-      libjpeg-turbo \
-      libxslt \
+      libpq5 \
       openssl \
-      postgresql-client \
-      postgresql-libs \
-      py3-pip \
       python3 \
+      python3-distutils \
       tini \
-      unit \
-      unit-python3
+    && curl -sL https://nginx.org/keys/nginx_signing.key \
+      > /etc/apt/trusted.gpg.d/nginx.asc && \
+    echo "deb https://packages.nginx.org/unit/debian/ bullseye unit" \
+      > /etc/apt/sources.list.d/unit.list \
+    && apt-get update -qq \
+    && apt-get install \
+      --yes -qq --no-install-recommends \
+      unit=1.27.0-1~bullseye \
+      unit-python3.9=1.27.0-1~bullseye \
+    && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /opt
-
-COPY --from=builder /usr/local/lib/libcrc32c.* /usr/local/lib/
-COPY --from=builder /usr/local/include/crc32c /usr/local/include
-COPY --from=builder /usr/local/lib/cmake/Crc32c /usr/local/lib/cmake/
 COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
+COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
 COPY docker/housekeeping.sh /opt/netbox/housekeeping.sh
 COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
@@ -97,30 +81,19 @@ WORKDIR /opt/netbox/netbox
 RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
       && chown -R unit:root media /opt/unit/ \
       && chmod -R g+w media /opt/unit/ \
-      && cd /opt/netbox/ && /opt/netbox/venv/bin/python -m mkdocs build \
+      && cd /opt/netbox/ && SECRET_KEY="dummy" /opt/netbox/venv/bin/python -m mkdocs build \
           --config-file /opt/netbox/mkdocs.yml --site-dir /opt/netbox/netbox/project-static/docs/ \
       && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
-ENTRYPOINT [ "/sbin/tini", "--" ]
+ENV LANG=C.UTF-8
+ENTRYPOINT [ "/usr/bin/tini", "--" ]
 
 CMD [ "/opt/netbox/docker-entrypoint.sh", "/opt/netbox/launch-netbox.sh" ]
 
-LABEL ORIGINAL_TAG="" \
-      NETBOX_GIT_BRANCH="" \
-      NETBOX_GIT_REF="" \
-      NETBOX_GIT_URL="" \
-# See http://label-schema.org/rc1/#build-time-labels
-# Also https://microbadger.com/labels
-      org.label-schema.schema-version="1.0" \
-      org.label-schema.build-date="" \
-      org.label-schema.name="NetBox Docker" \
-      org.label-schema.description="A container based distribution of NetBox, the free and open IPAM and DCIM solution." \
-      org.label-schema.vendor="The netbox-docker contributors." \
-      org.label-schema.url="https://github.com/netbox-community/netbox-docker" \
-      org.label-schema.usage="https://github.com/netbox-community/netbox-docker/wiki" \
-      org.label-schema.vcs-url="https://github.com/netbox-community/netbox-docker.git" \
-      org.label-schema.vcs-ref="" \
-      org.label-schema.version="snapshot" \
+LABEL netbox.original-tag="" \
+      netbox.git-branch="" \
+      netbox.git-ref="" \
+      netbox.git-url="" \
 # See https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys
       org.opencontainers.image.created="" \
       org.opencontainers.image.title="NetBox Docker" \
@@ -132,17 +105,4 @@ LABEL ORIGINAL_TAG="" \
       org.opencontainers.image.documentation="https://github.com/netbox-community/netbox-docker/wiki" \
       org.opencontainers.image.source="https://github.com/netbox-community/netbox-docker.git" \
       org.opencontainers.image.revision="" \
-      org.opencontainers.image.version="snapshot"
-
-#####
-## LDAP specific configuration
-#####
-
-FROM main as ldap
-
-RUN apk add --no-cache \
-      libsasl \
-      libldap \
-      util-linux
-
-COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
+      org.opencontainers.image.version=""

README.md

@@ -7,7 +7,7 @@
 ![Docker Pulls](https://img.shields.io/docker/pulls/netboxcommunity/netbox)
 [![GitHub license](https://img.shields.io/github/license/netbox-community/netbox-docker)][netbox-docker-license]
 
-[The Github repository](netbox-docker-github) houses the components needed to build NetBox as a container.
+[The GitHub repository](netbox-docker-github) houses the components needed to build NetBox as a container.
 Images are built regularly using the code in that repository and are pushed to [Docker Hub][netbox-dockerhub], [Quay.io][netbox-quayio] and [GitHub Container Registry][netbox-ghcr].
 
 Do you have any questions?
@@ -19,7 +19,7 @@ please join [our Slack][netbox-docker-slack] and ask for help in the [`#netbox-d
 [netbox-docker-microbadger]: https://microbadger.com/images/netboxcommunity/netbox
 [netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/
 [netbox-quayio]: https://quay.io/repository/netboxcommunity/netbox
-[netbox-ghcr]: https://ghcr.io/netbox-community/netbox/
+[netbox-ghcr]: https://github.com/netbox-community/netbox-docker/pkgs/container/netbox
 [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/
 [netbox-docker-slack]: https://join.slack.com/t/netdev-community/shared_invite/zt-mtts8g0n-Sm6Wutn62q_M4OdsaIycrQ
 [netbox-docker-slack-channel]: https://netdev-community.slack.com/archives/C01P0GEVBU7
@@ -75,7 +75,7 @@ New container images are built and published automatically every ~24h.
   You must use _NetBox Docker version_ `a.b.c` to guarantee the compatibility.
   These images are automatically built from [the `master` branch of NetBox][netbox-master].
 * `snapshot-a.b.c`:
-  These are pre-release builds.
+  These are prerelease builds.
   They contain the support files of _NetBox Docker version_ `a.b.c`.
   You must use _NetBox Docker version_ `a.b.c` to guarantee the compatibility.
   These images are automatically built from the [`develop` branch of NetBox][netbox-develop].
@@ -106,7 +106,7 @@ Then there is currently one extra tags for each of the above tags:
 
 ## Documentation
 
-Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use the NetBox Docker image properly.
+Please refer [to our wiki on GitHub][netbox-docker-wiki] for further information on how to use the NetBox Docker image properly.
 The wiki covers advanced topics such as using files for secrets, configuring TLS, deployment to Kubernetes, monitoring and configuring NAPALM and LDAP.
 
 Our wiki is a community effort.
@@ -116,7 +116,7 @@ Feel free to correct errors, update outdated information or provide additional g
 
 ## Getting Help
 
-Feel free to ask questions in our [Github Community][netbox-community]
+Feel free to ask questions in our [GitHub Community][netbox-community]
 or [join our Slack][netbox-docker-slack] and ask [in our channel `#netbox-docker`][netbox-docker-slack-channel],
 which is free to use and where there are almost always people online that can help you in the Slack channel.
 

VERSION

@@ -1 +1 @@
-1.5.0
+2.0.0

build.sh

@@ -49,10 +49,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: 'alpine:3.14'"
+  echo "              Default: 'debian:11-slim'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
-  echo "              Default: main ldap"
+  echo "              Default: main"
   echo "  HTTP_PROXY  The proxy to use for http requests."
   echo "              Example: http://proxy.domain.tld:3128"
   echo "              Default: undefined"
@@ -170,7 +170,7 @@ fi
 # Determining the value for DOCKER_FROM
 ###
 if [ -z "$DOCKER_FROM" ]; then
-  DOCKER_FROM="alpine:3.14"
+  DOCKER_FROM="debian:11-slim"
 fi
 
 ###
@@ -178,7 +178,7 @@ fi
 ###
 BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')"
 
-if [ -d ".git" ]; then
+if [ -d ".git" ] && [ -z "${SKIP_GIT}" ]; then
   GIT_REF="$(git rev-parse HEAD)"
 fi
 
@@ -186,7 +186,7 @@ fi
 PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
 
 # Get the Git information from the netbox directory
-if [ -d "${NETBOX_PATH}/.git" ]; then
+if [ -d "${NETBOX_PATH}/.git" ] && [ -z "${SKIP_GIT}" ]; then
   NETBOX_GIT_REF=$(
     cd "${NETBOX_PATH}"
     git rev-parse HEAD
@@ -222,7 +222,7 @@ esac
 ###
 # Determine targets to build
 ###
-DEFAULT_DOCKER_TARGETS=("main" "ldap")
+DEFAULT_DOCKER_TARGETS=("main")
 DOCKER_TARGETS=("${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}")
 echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
 
@@ -277,9 +277,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     ###
     # Checking if the build is necessary,
     # meaning build only if one of those values changed:
-    # - Python base image digest (Label: PYTHON_BASE_DIGEST)
-    # - netbox git ref (Label: NETBOX_GIT_REF)
-    # - netbox-docker git ref (Label: org.label-schema.vcs-ref)
+    # - base image digest
+    # - netbox git ref (Label: netbox.git-ref)
+    # - netbox-docker git ref (Label: org.opencontainers.image.revision)
     ###
     # Load information from registry (only for docker.io)
     SHOULD_BUILD="false"
@@ -295,14 +295,14 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
         # Need to use "library/..." for images the have no two part name
         DOCKER_FROM_SPLIT[0]="library/${DOCKER_FROM_SPLIT[0]}"
       fi
-      PYTHON_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}")
+      BASE_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}")
       mapfile -t IMAGES_LAYERS_OLD < <(get_image_layers "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
-      NETBOX_GIT_REF_OLD=$(get_image_label NETBOX_GIT_REF "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
-      GIT_REF_OLD=$(get_image_label org.label-schema.vcs-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
+      NETBOX_GIT_REF_OLD=$(get_image_label netbox.git-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
+      GIT_REF_OLD=$(get_image_label org.opencontainers.image.revision "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
 
-      if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then
+      if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${BASE_LAST_LAYER}\$"; then
         SHOULD_BUILD="true"
-        BUILD_REASON="${BUILD_REASON} alpine"
+        BUILD_REASON="${BUILD_REASON} debian"
       fi
       if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then
         SHOULD_BUILD="true"
@@ -335,30 +335,25 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
 
     # --label
     DOCKER_BUILD_ARGS+=(
-      --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG_PROJECT}"
-
-      --label "org.label-schema.build-date=${BUILD_DATE}"
+      --label "netbox.original-tag=${TARGET_DOCKER_TAG_PROJECT}"
       --label "org.opencontainers.image.created=${BUILD_DATE}"
-
-      --label "org.label-schema.version=${PROJECT_VERSION}"
       --label "org.opencontainers.image.version=${PROJECT_VERSION}"
     )
     if [ -d ".git" ]; then
       DOCKER_BUILD_ARGS+=(
-        --label "org.label-schema.vcs-ref=${GIT_REF}"
         --label "org.opencontainers.image.revision=${GIT_REF}"
       )
     fi
     if [ -d "${NETBOX_PATH}/.git" ]; then
       DOCKER_BUILD_ARGS+=(
-        --label "NETBOX_GIT_BRANCH=${NETBOX_GIT_BRANCH}"
-        --label "NETBOX_GIT_REF=${NETBOX_GIT_REF}"
-        --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}"
+        --label "netbox.git-branch=${NETBOX_GIT_BRANCH}"
+        --label "netbox.git-ref=${NETBOX_GIT_REF}"
+        --label "netbox.git-url=${NETBOX_GIT_URL}"
       )
     fi
     if [ -n "${BUILD_REASON}" ]; then
       BUILD_REASON=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<"$BUILD_REASON")
-      DOCKER_BUILD_ARGS+=(--label "BUILD_REASON=${BUILD_REASON}")
+      DOCKER_BUILD_ARGS+=(--label "netbox.build-reason=${BUILD_REASON}")
     fi
 
     # --build-arg
@@ -385,7 +380,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
       $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
       echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG_PROJECT}'"
       echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG_PROJECT}'"
-      $DRY docker inspect "${TARGET_DOCKER_TAG_PROJECT}" --format "{{json .Config.Labels}}"
+      $DRY docker inspect "${TARGET_DOCKER_TAG_PROJECT}" --format "{{json .Config.Labels}}" | jq
     else
       echo "Build skipped because sources didn't change"
       echo "::set-output name=skipped::true"

configuration/configuration.py

@@ -120,6 +120,11 @@ CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() ==
 CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' ')))
 CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))]
 
+# Cross-Site-Request-Forgery-Attack settings. If Netbox is sitting behind a reverse proxy, you might need to set the CSRF_TRUSTED_ORIGINS flag.
+# Django 4.0 requires to specify the URL Scheme in this setting. An example environment variable could be specified like:
+# CSRF_TRUSTED_ORIGINS=https://demo.netbox.dev http://demo.netbox.dev
+CSRF_TRUSTED_ORIGINS = list(filter(None, environ.get('CSRF_TRUSTED_ORIGINS', '').split(' ')))
+
 # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal
 # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging
 # on a production system.
@@ -147,6 +152,9 @@ ENFORCE_GLOBAL_UNIQUE = environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() ==
 # by anonymous users. List models in the form `<app>.<model>`. Add '*' to this list to exempt all models.
 EXEMPT_VIEW_PERMISSIONS = list(filter(None, environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' ')))
 
+# Enable GraphQL API.
+GRAPHQL_ENABLED = environ.get('GRAPHQL_ENABLED', 'True').lower() == 'true'
+
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
 #   https://docs.djangoproject.com/en/stable/topics/logging/
 LOGGING = {}
@@ -162,6 +170,9 @@ LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600))
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
 MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true'
 
+# Maps provider
+MAPS_URL = environ.get('MAPS_URL', None)
+
 # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g.
 # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request
 # all objects by specifying "?limit=0".

docker-compose.test.yml

@@ -21,14 +21,14 @@ services:
     image: postgres:14-alpine
     env_file: env/postgres.env
   redis:
-    image: redis:6-alpine
+    image: redis:7-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
     - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
     env_file: env/redis.env
   redis-cache:
-    image: redis:6-alpine
+    image: redis:7-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.4'
 services:
   netbox: &netbox
-    image: netboxcommunity/netbox:${VERSION-v3.1-1.5.0}
+    image: netboxcommunity/netbox:${VERSION-v3.2-2.0.0}
     depends_on:
     - postgres
     - redis
@@ -42,7 +42,7 @@ services:
 
   # redis
   redis:
-    image: redis:6-alpine
+    image: redis:7-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
@@ -51,7 +51,7 @@ services:
     volumes:
     - netbox-redis-data:/data
   redis-cache:
-    image: redis:6-alpine
+    image: redis:7-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env

docker/nginx-unit.json

@@ -11,7 +11,7 @@
         "uri": "/static/*"
       },
       "action": {
-        "share": "/opt/netbox/netbox"
+        "share": "/opt/netbox/netbox${uri}"
       }
     },
 

env/netbox.env

@@ -14,6 +14,7 @@ EMAIL_USERNAME=netbox
 # EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`!
 EMAIL_USE_SSL=false
 EMAIL_USE_TLS=false
+GRAPHQL_ENABLED=true
 HOUSEKEEPING_INTERVAL=86400
 MAX_PAGE_SIZE=1000
 MEDIA_ROOT=/opt/netbox/netbox/media

initializers/asns.yml

@@ -0,0 +1,7 @@
+# - asn: 1
+#   rir: RFC1918
+#   tenant: tenant1
+# - asn: 2
+#   rir: RFC4193 ULA
+# - asn: 3
+#   rir: RFC3849

initializers/cables.yml

@@ -0,0 +1,71 @@
+# # Required parameters for termination X ('a' or 'b'):
+# #
+# # ```
+# #   termination_x_name -> name of interface
+# #   termination_x_device -> name of the device interface belongs to
+# #   termination_x_class -> required if different than 'Interface' which is the default
+# # ```
+# #
+# # Supported termination classes: Interface, ConsolePort, ConsoleServerPort, FrontPort, RearPort, PowerPort, PowerOutlet
+# #
+# #
+# # If a termination is a circuit then the required parameter is termination_x_circuit.
+# # Required parameters for a circuit termination:
+# #
+# # ```
+# # termination_x_circuit:
+# #   term_side -> termination side of a circuit. Must be A or B
+# #   cid -> circuit ID value
+# #   site OR provider_network -> name of Site or ProviderNetwork respectively. If both provided, Site takes precedence
+# # ```
+# #
+# # If a termination is a power feed then the required parameter is termination_x_feed.
+# #
+# # ```
+# #   termination_x_feed:
+# #     name -> name of the PowerFeed object
+# #     power_panel:
+# #       name -> name of the PowerPanel the PowerFeed is attached to
+# #       site -> name of the Site in which the PowerPanel is present
+# # ```
+# #
+# # Any other Cable parameters supported by Netbox are supported as the top level keys, e.g. 'type', 'status', etc.
+# #
+# # - termination_a_name: console
+# #   termination_a_device: spine
+# #   termination_a_class: ConsolePort
+# #   termination_b_name: tty9
+# #   termination_b_device: console-server
+# #   termination_b_class: ConsoleServerPort
+# #   type: cat6
+# #
+# - termination_a_name: to-server02
+#   termination_a_device: server01
+#   termination_b_name: to-server01
+#   termination_b_device: server02
+#   status: planned
+#   type: mmf
+
+# - termination_a_name: eth0
+#   termination_a_device: server02
+#   termination_b_circuit:
+#     term_side: A
+#     cid: Circuit_ID-1
+#     site: AMS 1
+#   type: cat6
+
+# - termination_a_name: psu0
+#   termination_a_device: server04
+#   termination_a_class: PowerPort
+#   termination_b_feed:
+#     name: power feed 1
+#     power_panel:
+#       name: power panel AMS 1
+#       site: AMS 1
+
+# - termination_a_name: outlet1
+#   termination_a_device: server04
+#   termination_a_class: PowerOutlet
+#   termination_b_name: psu1
+#   termination_b_device: server04
+#   termination_b_class: PowerPort

initializers/contact_groups.yml

@@ -0,0 +1,7 @@
+# - name: Network-Team
+#   slug: network-team
+#   description: This is a new contact group for the Network-Team
+# - name: New Contact Group
+#   slug: new-contact-group
+#   description: This is a new contact group sub under of Network-Team
+#   parent: Network-Team

initializers/contact_roles.yml

@@ -0,0 +1,3 @@
+# - name: New Contact Role
+#   slug: new-contact-role
+#   description: This is a new contact role description

initializers/contacts.yml

@@ -0,0 +1,20 @@
+# - name: Lee Widget
+#   title: CEO of Widget Corp
+#   phone: 221-555-1212
+#   email: widgetCEO@widgetcorp.com
+#   address: 1200 Nowhere Blvd, Scranton NJ, 555111
+#   comments: This is a very important contact
+# - name: Ali Gator
+#   group: Network-Team
+#   title: Consultant for Widget Corp
+#   phone: 221-555-1213
+#   email: Consultant@widgetcorp.com
+#   address: 1200 Nowhere Blvd, Scranton NJ, 555111
+#   comments: This is a very important contact
+# - name: Karlchen Maier
+#   group: New Contact Group
+#   title: COO of Widget Corp
+#   phone: 221-555-1214
+#   email: Karlchen@widgetcorp.com
+#   address: 1200 Nowhere Blvd, Scranton NJ, 555111
+#   comments: This is a very important contact

initializers/dcim_interfaces.yml

@@ -8,11 +8,28 @@
 ##
 ## Examples:
 
+# - device: server01
+#   name: ath0
+#   type: 1000base-t
+#   lag: ae0
+#   bridge: br0
+# - device: server01
+#   name: ath1
+#   type: 1000base-t
+#   parent: ath0
 # - device: server01
 #   enabled: true
-#   type: virtual
+#   type: 1000base-x-sfp
 #   name: to-server02
 # - device: server02
 #   enabled: true
-#   type: virtual
+#   type: 1000base-x-sfp
 #   name: to-server01
+# - device: server02
+#   enabled: true
+#   type: 1000base-t
+#   name: eth0
+# - device: server02
+#   enabled: true
+#   type: virtual
+#   name: loopback

initializers/device_types.yml

@@ -21,3 +21,37 @@
 #   slug: other
 #   custom_field_data:
 #     text_field: Description
+#   interfaces:
+#     - name: eth0
+#       type: 1000base-t
+#       mgmt_only: True
+#     - name: eth1
+#       type: 1000base-t
+#   console_server_ports:
+#     - name_template: ttyS[1-48]
+#       type: rj-45
+#   power_ports:
+#     - name_template: psu[0,1]
+#       type: iec-60320-c14
+#       maximum_draw: 35
+#       allocated_draw: 35
+#   front_ports:
+#     - name_template: front[1,2]
+#       type: 8p8c
+#       rear_port_template: rear[0,1]
+#       rear_port_position_template: "[1,2]"
+#   rear_ports:
+#     - name_template: rear[0,1]
+#       type: 8p8c
+#       positions_template: "[3,2]"
+#   device_bays:
+#     - name: bay0 # both non-template and template field specified; non-template field takes precedence
+#       name_template: bay[0-9]
+#       label: test0
+#       label_template: test[0-5,9,6-8]
+#       description: Test description
+#   power_outlets:
+#     - name_template: outlet[0,1]
+#       type: iec-60320-c5
+#       power_port: psu0
+#       feed_leg: B

initializers/sites.yml

@@ -3,7 +3,6 @@
 #   region: Downtown
 #   status: active
 #   facility: Amsterdam 1
-#   asn: 12345
 #   custom_field_data:
 #     text_field: Description for AMS1
 # - name: AMS 2
@@ -11,7 +10,6 @@
 #   region: Downtown
 #   status: active
 #   facility: Amsterdam 2
-#   asn: 54321
 #   custom_field_data:
 #     text_field: Description for AMS2
 # - name: AMS 3
@@ -19,7 +17,6 @@
 #   region: Suburbs
 #   status: active
 #   facility: Amsterdam 3
-#   asn: 67890
 #   tenant: tenant1
 #   custom_field_data:
 #     text_field: Description for AMS3
@@ -28,7 +25,6 @@
 #   region: Singapore
 #   status: active
 #   facility: Singapore 1
-#   asn: 09876
 #   tenant: tenant2
 #   custom_field_data:
 #     text_field: Description for SING1

initializers/users.yml

@@ -4,6 +4,7 @@
 #   password: reader
 # writer:
 #   password: writer
+#   api_token: "" # a token is generated automatically unless the value is explicity set to empty
 # jdoe:
 #   first_name: John
 #   last_name: Doe

requirements-container.txt

@@ -1,5 +1,5 @@
-napalm==3.3.1
-ruamel.yaml==0.17.17
-django-auth-ldap==3.0.0
-google-crc32c==1.3.0
+django-auth-ldap==4.1.0
 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.12.3
+napalm==4.0.0
+psycopg2==2.9.3
+ruamel.yaml==0.17.21

startup_scripts/000_users.py

@@ -9,13 +9,17 @@ if users is None:
     sys.exit()
 
 for username, user_details in users.items():
-    if not User.objects.filter(username=username):
-        user = User.objects.create_user(
-            username=username,
-            password=user_details.get("password", 0) or User.objects.make_random_password(),
-        )
+
+    api_token = user_details.pop("api_token", Token.generate_key())
+    password = user_details.pop("password", User.objects.make_random_password())
+
+    user, created = User.objects.get_or_create(username=username, defaults=user_details)
+
+    if created:
+        user.set_password(password)
+        user.save()
+
+        if api_token:
+            Token.objects.get_or_create(user=user, key=api_token)
 
         print("👤 Created user", username)
-
-        if user_details.get("api_token", 0):
-            Token.objects.create(user=user, key=user_details["api_token"])

startup_scripts/020_object_permissions.py

@@ -14,9 +14,11 @@ for permission_name, permission_details in object_permissions.items():
 
     object_permission, created = ObjectPermission.objects.get_or_create(
         name=permission_name,
-        description=permission_details["description"],
-        enabled=permission_details["enabled"],
-        actions=permission_details["actions"],
+        defaults={
+            "description": permission_details["description"],
+            "enabled": permission_details["enabled"],
+            "actions": permission_details["actions"],
+        },
     )
 
     if permission_details.get("object_types", 0):

startup_scripts/040_custom_links.py

@@ -2,7 +2,7 @@ import sys
 
 from django.contrib.contenttypes.models import ContentType
 from extras.models import CustomLink
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 custom_links = load_yaml("/opt/netbox/initializers/custom_links.yml")
 
@@ -28,6 +28,8 @@ for link in custom_links:
         )
         continue
 
-    custom_link, created = CustomLink.objects.get_or_create(**link)
+    matching_params, defaults = split_params(link)
+    custom_link, created = CustomLink.objects.get_or_create(**matching_params, defaults=defaults)
+
     if created:
         print("🔗 Created Custom Link '{0}'".format(custom_link.name))

startup_scripts/050_tags.py

@@ -1,7 +1,7 @@
 import sys
 
 from extras.models import Tag
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from utilities.choices import ColorChoices
 
 tags = load_yaml("/opt/netbox/initializers/tags.yml")
@@ -17,7 +17,8 @@ for params in tags:
             if color in color_tpl:
                 params["color"] = color_tpl[0]
 
-    tag, created = Tag.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    tag, created = Tag.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🎨 Created Tag", tag.name)

startup_scripts/060_device_types.py

@@ -1,37 +0,0 @@
-import sys
-
-from dcim.models import DeviceType, Manufacturer, Region
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
-from tenancy.models import Tenant
-
-device_types = load_yaml("/opt/netbox/initializers/device_types.yml")
-
-if device_types is None:
-    sys.exit()
-
-required_assocs = {"manufacturer": (Manufacturer, "name")}
-
-optional_assocs = {"region": (Region, "name"), "tenant": (Tenant, "name")}
-
-for params in device_types:
-    custom_field_data = pop_custom_fields(params)
-
-    for assoc, details in required_assocs.items():
-        model, field = details
-        query = {field: params.pop(assoc)}
-
-        params[assoc] = model.objects.get(**query)
-
-    for assoc, details in optional_assocs.items():
-        if assoc in params:
-            model, field = details
-            query = {field: params.pop(assoc)}
-
-            params[assoc] = model.objects.get(**query)
-
-    device_type, created = DeviceType.objects.get_or_create(**params)
-
-    if created:
-        set_custom_fields_values(device_type, custom_field_data)
-
-        print("🔡 Created device type", device_type.manufacturer, device_type.model)

startup_scripts/060_webhooks.py

@@ -2,7 +2,7 @@ import sys
 
 from django.contrib.contenttypes.models import ContentType
 from extras.models import Webhook
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 webhooks = load_yaml("/opt/netbox/initializers/webhooks.yml")
 
@@ -26,7 +26,9 @@ for hook in webhooks:
     except ContentType.DoesNotExist:
         continue
 
-    webhook, created = Webhook.objects.get_or_create(**hook)
+    matching_params, defaults = split_params(hook)
+    webhook, created = Webhook.objects.get_or_create(**matching_params, defaults=defaults)
+
     if created:
         webhook.content_types.set(obj_type_ids)
         webhook.save()

startup_scripts/070_tenant_groups.py

@@ -1,6 +1,6 @@
 import sys
 
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from tenancy.models import TenantGroup
 
 tenant_groups = load_yaml("/opt/netbox/initializers/tenant_groups.yml")
@@ -9,7 +9,8 @@ if tenant_groups is None:
     sys.exit()
 
 for params in tenant_groups:
-    tenant_group, created = TenantGroup.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    tenant_group, created = TenantGroup.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🔳 Created Tenant Group", tenant_group.name)

startup_scripts/080_tenants.py

@@ -1,6 +1,11 @@
 import sys
 
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant, TenantGroup
 
 tenants = load_yaml("/opt/netbox/initializers/tenants.yml")
@@ -20,9 +25,10 @@ for params in tenants:
 
             params[assoc] = model.objects.get(**query)
 
-    tenant, created = Tenant.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    tenant, created = Tenant.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(tenant, custom_field_data)
-
         print("👩‍💻 Created Tenant", tenant.name)
+
+    set_custom_fields_values(tenant, custom_field_data)

startup_scripts/090_regions.py

@@ -1,7 +1,7 @@
 import sys
 
 from dcim.models import Region
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 regions = load_yaml("/opt/netbox/initializers/regions.yml")
 
@@ -19,7 +19,8 @@ for params in regions:
 
             params[assoc] = model.objects.get(**query)
 
-    region, created = Region.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    region, created = Region.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🌐 Created region", region.name)

startup_scripts/110_sites.py

@@ -1,7 +1,12 @@
 import sys
 
 from dcim.models import Region, Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 
 sites = load_yaml("/opt/netbox/initializers/sites.yml")
@@ -21,9 +26,10 @@ for params in sites:
 
             params[assoc] = model.objects.get(**query)
 
-    site, created = Site.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    site, created = Site.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(site, custom_field_data)
-
         print("📍 Created site", site.name)
+
+    set_custom_fields_values(site, custom_field_data)

startup_scripts/120_locations.py

@@ -1,13 +1,14 @@
 import sys
 
 from dcim.models import Location, Site
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 rack_groups = load_yaml("/opt/netbox/initializers/locations.yml")
 
 if rack_groups is None:
     sys.exit()
 
+match_params = ["name", "slug", "site"]
 required_assocs = {"site": (Site, "name")}
 
 for params in rack_groups:
@@ -17,7 +18,8 @@ for params in rack_groups:
         query = {field: params.pop(assoc)}
         params[assoc] = model.objects.get(**query)
 
-    location, created = Location.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    location, created = Location.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🎨 Created location", location.name)

startup_scripts/130_rack_roles.py

@@ -1,7 +1,7 @@
 import sys
 
 from dcim.models import RackRole
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from utilities.choices import ColorChoices
 
 rack_roles = load_yaml("/opt/netbox/initializers/rack_roles.yml")
@@ -17,7 +17,8 @@ for params in rack_roles:
             if color in color_tpl:
                 params["color"] = color_tpl[0]
 
-    rack_role, created = RackRole.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    rack_role, created = RackRole.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🎨 Created rack role", rack_role.name)

startup_scripts/135_cluster_groups.py

@@ -1,15 +0,0 @@
-import sys
-
-from startup_script_utils import load_yaml
-from virtualization.models import ClusterGroup
-
-cluster_groups = load_yaml("/opt/netbox/initializers/cluster_groups.yml")
-
-if cluster_groups is None:
-    sys.exit()
-
-for params in cluster_groups:
-    cluster_group, created = ClusterGroup.objects.get_or_create(**params)
-
-    if created:
-        print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/140_clusters.py

@@ -1,42 +0,0 @@
-import sys
-
-from dcim.models import Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
-from tenancy.models import Tenant
-from virtualization.models import Cluster, ClusterGroup, ClusterType
-
-clusters = load_yaml("/opt/netbox/initializers/clusters.yml")
-
-if clusters is None:
-    sys.exit()
-
-required_assocs = {"type": (ClusterType, "name")}
-
-optional_assocs = {
-    "site": (Site, "name"),
-    "group": (ClusterGroup, "name"),
-    "tenant": (Tenant, "name"),
-}
-
-for params in clusters:
-    custom_field_data = pop_custom_fields(params)
-
-    for assoc, details in required_assocs.items():
-        model, field = details
-        query = {field: params.pop(assoc)}
-
-        params[assoc] = model.objects.get(**query)
-
-    for assoc, details in optional_assocs.items():
-        if assoc in params:
-            model, field = details
-            query = {field: params.pop(assoc)}
-
-            params[assoc] = model.objects.get(**query)
-
-    cluster, created = Cluster.objects.get_or_create(**params)
-
-    if created:
-        set_custom_fields_values(cluster, custom_field_data)
-
-        print("🗄️ Created cluster", cluster.name)

startup_scripts/140_racks.py

@@ -1,7 +1,12 @@
 import sys
 
 from dcim.models import Location, Rack, RackRole, Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 
 racks = load_yaml("/opt/netbox/initializers/racks.yml")
@@ -9,8 +14,8 @@ racks = load_yaml("/opt/netbox/initializers/racks.yml")
 if racks is None:
     sys.exit()
 
+match_params = ["name", "site"]
 required_assocs = {"site": (Site, "name")}
-
 optional_assocs = {
     "role": (RackRole, "name"),
     "tenant": (Tenant, "name"),
@@ -33,9 +38,10 @@ for params in racks:
 
             params[assoc] = model.objects.get(**query)
 
-    rack, created = Rack.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    rack, created = Rack.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(rack, custom_field_data)
-
         print("🔳 Created rack", rack.site, rack.name)
+
+    set_custom_fields_values(rack, custom_field_data)

startup_scripts/150_power_panels.py

@@ -1,15 +1,20 @@
 import sys
 
 from dcim.models import Location, PowerPanel, Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 
 power_panels = load_yaml("/opt/netbox/initializers/power_panels.yml")
 
 if power_panels is None:
     sys.exit()
 
+match_params = ["name", "site"]
 required_assocs = {"site": (Site, "name")}
-
 optional_assocs = {"location": (Location, "name")}
 
 for params in power_panels:
@@ -28,9 +33,10 @@ for params in power_panels:
 
             params[assoc] = model.objects.get(**query)
 
-    power_panel, created = PowerPanel.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    power_panel, created = PowerPanel.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(power_panel, custom_field_data)
-
         print("⚡ Created Power Panel", power_panel.site, power_panel.name)
+
+    set_custom_fields_values(power_panel, custom_field_data)

startup_scripts/160_power_feeds.py

@@ -1,15 +1,20 @@
 import sys
 
 from dcim.models import PowerFeed, PowerPanel, Rack
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 
 power_feeds = load_yaml("/opt/netbox/initializers/power_feeds.yml")
 
 if power_feeds is None:
     sys.exit()
 
+match_params = ["name", "power_panel"]
 required_assocs = {"power_panel": (PowerPanel, "name")}
-
 optional_assocs = {"rack": (Rack, "name")}
 
 for params in power_feeds:
@@ -28,9 +33,10 @@ for params in power_feeds:
 
             params[assoc] = model.objects.get(**query)
 
-    power_feed, created = PowerFeed.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    power_feed, created = PowerFeed.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(power_feed, custom_field_data)
-
         print("⚡ Created Power Feed", power_feed.name)
+
+    set_custom_fields_values(power_feed, custom_field_data)

startup_scripts/170_manufacturers.py

@@ -1,7 +1,7 @@
 import sys
 
 from dcim.models import Manufacturer
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 manufacturers = load_yaml("/opt/netbox/initializers/manufacturers.yml")
 
@@ -9,7 +9,8 @@ if manufacturers is None:
     sys.exit()
 
 for params in manufacturers:
-    manufacturer, created = Manufacturer.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    manufacturer, created = Manufacturer.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🏭 Created Manufacturer", manufacturer.name)

startup_scripts/180_device_roles.py

@@ -1,7 +1,7 @@
 import sys
 
 from dcim.models import DeviceRole
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from utilities.choices import ColorChoices
 
 device_roles = load_yaml("/opt/netbox/initializers/device_roles.yml")
@@ -18,7 +18,8 @@ for params in device_roles:
             if color in color_tpl:
                 params["color"] = color_tpl[0]
 
-    device_role, created = DeviceRole.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    device_role, created = DeviceRole.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🎨 Created device role", device_role.name)

startup_scripts/190_device_types.py

@@ -0,0 +1,139 @@
+import sys
+from typing import List
+
+from dcim.models import DeviceType, Manufacturer, Region
+from dcim.models.device_component_templates import (
+    ConsolePortTemplate,
+    ConsoleServerPortTemplate,
+    DeviceBayTemplate,
+    FrontPortTemplate,
+    InterfaceTemplate,
+    PowerOutletTemplate,
+    PowerPortTemplate,
+    RearPortTemplate,
+)
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
+from tenancy.models import Tenant
+from utilities.forms.utils import expand_alphanumeric_pattern
+
+
+def expand_templates(params: List[dict], device_type: DeviceType) -> List[dict]:
+    templateable_fields = ["name", "label", "positions", "rear_port", "rear_port_position"]
+
+    expanded = []
+    for param in params:
+        param["device_type"] = device_type
+        expanded_fields = {}
+        has_plain_fields = False
+
+        for field in templateable_fields:
+            template_value = param.pop(f"{field}_template", None)
+
+            if field in param:
+                has_plain_fields = True
+            elif template_value:
+                expanded_fields[field] = list(expand_alphanumeric_pattern(template_value))
+
+        if expanded_fields and has_plain_fields:
+            raise ValueError(f"Mix of plain and template keys provided for {templateable_fields}")
+        elif not expanded_fields:
+            expanded.append(param)
+            continue
+
+        elements = list(expanded_fields.values())
+        master_len = len(elements[0])
+        if not all([len(elem) == master_len for elem in elements]):
+            raise ValueError(
+                f"Number of elements in template fields "
+                f"{list(expanded_fields.keys())} must be equal"
+            )
+
+        for idx in range(master_len):
+            tmp = param.copy()
+            for field, value in expanded_fields.items():
+                if field in nested_assocs:
+                    model, match_key = nested_assocs[field]
+                    query = {match_key: value[idx], "device_type": device_type}
+                    tmp[field] = model.objects.get(**query)
+                else:
+                    tmp[field] = value[idx]
+            expanded.append(tmp)
+    return expanded
+
+
+device_types = load_yaml("/opt/netbox/initializers/device_types.yml")
+
+if device_types is None:
+    sys.exit()
+
+match_params = ["manufacturer", "model", "slug"]
+required_assocs = {"manufacturer": (Manufacturer, "name")}
+optional_assocs = {"region": (Region, "name"), "tenant": (Tenant, "name")}
+nested_assocs = {"rear_port": (RearPortTemplate, "name"), "power_port": (PowerPortTemplate, "name")}
+
+supported_components = {
+    "interfaces": (InterfaceTemplate, ["name"]),
+    "console_ports": (ConsolePortTemplate, ["name"]),
+    "console_server_ports": (ConsoleServerPortTemplate, ["name"]),
+    "power_ports": (PowerPortTemplate, ["name"]),
+    "power_outlets": (PowerOutletTemplate, ["name"]),
+    "rear_ports": (RearPortTemplate, ["name"]),
+    "front_ports": (FrontPortTemplate, ["name"]),
+    "device_bays": (DeviceBayTemplate, ["name"]),
+}
+
+for params in device_types:
+    custom_field_data = pop_custom_fields(params)
+    components = [(v[0], v[1], params.pop(k, [])) for k, v in supported_components.items()]
+
+    for assoc, details in required_assocs.items():
+        model, field = details
+        query = {field: params.pop(assoc)}
+
+        params[assoc] = model.objects.get(**query)
+
+    for assoc, details in optional_assocs.items():
+        if assoc in params:
+            model, field = details
+            query = {field: params.pop(assoc)}
+
+            params[assoc] = model.objects.get(**query)
+
+    matching_params, defaults = split_params(params, match_params)
+    device_type, created = DeviceType.objects.get_or_create(**matching_params, defaults=defaults)
+
+    if created:
+        print("🔡 Created device type", device_type.manufacturer, device_type.model)
+
+    set_custom_fields_values(device_type, custom_field_data)
+
+    for component in components:
+        c_model, c_match_params, c_params = component
+        c_match_params.append("device_type")
+
+        if not c_params:
+            continue
+
+        expanded_c_params = expand_templates(c_params, device_type)
+
+        for n_assoc, n_details in nested_assocs.items():
+            n_model, n_field = n_details
+            for c_param in expanded_c_params:
+                if n_assoc in c_param:
+                    n_query = {n_field: c_param[n_assoc], "device_type": device_type}
+                    c_param[n_assoc] = n_model.objects.get(**n_query)
+
+        for new_param in expanded_c_params:
+            new_matching_params, new_defaults = split_params(new_param, c_match_params)
+            new_obj, new_obj_created = c_model.objects.get_or_create(
+                **new_matching_params, defaults=new_defaults
+            )
+            if new_obj_created:
+                print(
+                    f"🧷  Created {c_model._meta} {new_obj} component for device type {device_type}"
+                )

startup_scripts/200_devices.py

@@ -1,7 +1,12 @@
 import sys
 
 from dcim.models import Device, DeviceRole, DeviceType, Location, Platform, Rack, Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 from virtualization.models import Cluster
 
@@ -10,12 +15,12 @@ devices = load_yaml("/opt/netbox/initializers/devices.yml")
 if devices is None:
     sys.exit()
 
+match_params = ["device_type", "name", "site"]
 required_assocs = {
     "device_role": (DeviceRole, "name"),
     "device_type": (DeviceType, "model"),
     "site": (Site, "name"),
 }
-
 optional_assocs = {
     "tenant": (Tenant, "name"),
     "platform": (Platform, "name"),
@@ -27,7 +32,7 @@ optional_assocs = {
 for params in devices:
     custom_field_data = pop_custom_fields(params)
 
-    # primary ips are handled later in `270_primary_ips.py`
+    # primary ips are handled later in `380_primary_ips.py`
     params.pop("primary_ip4", None)
     params.pop("primary_ip6", None)
 
@@ -44,9 +49,10 @@ for params in devices:
 
             params[assoc] = model.objects.get(**query)
 
-    device, created = Device.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    device, created = Device.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(device, custom_field_data)
-
         print("🖥️  Created device", device.name)
+
+    set_custom_fields_values(device, custom_field_data)

startup_scripts/210_dcim_interfaces.py

@@ -0,0 +1,70 @@
+import sys
+
+from dcim.models import Device, Interface
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
+
+interfaces = load_yaml("/opt/netbox/initializers/dcim_interfaces.yml")
+
+if interfaces is None:
+    sys.exit()
+
+match_params = ["device", "name"]
+required_assocs = {"device": (Device, "name")}
+related_assocs = {
+    "bridge": (Interface, "name"),
+    "lag": (Interface, "name"),
+    "parent": (Interface, "name"),
+}
+
+for params in interfaces:
+    custom_field_data = pop_custom_fields(params)
+
+    related_interfaces = {k: params.pop(k, None) for k in related_assocs}
+
+    for assoc, details in required_assocs.items():
+        model, field = details
+        query = {field: params.pop(assoc)}
+
+        params[assoc] = model.objects.get(**query)
+
+    matching_params, defaults = split_params(params, match_params)
+    interface, created = Interface.objects.get_or_create(**matching_params, defaults=defaults)
+
+    if created:
+        print(f"🧷 Created interface {interface} on {interface.device}")
+
+    set_custom_fields_values(interface, custom_field_data)
+
+    for related_field, related_value in related_interfaces.items():
+        if not related_value:
+            continue
+
+        r_model, r_field = related_assocs[related_field]
+
+        if related_field == "parent" and not interface.parent_id:
+            query = {r_field: related_value, "device": interface.device}
+            try:
+                related_obj = r_model.objects.get(**query)
+            except Interface.DoesNotExist:
+                print(f"⚠️ Could not find parent interface with: {query} for interface {interface}")
+                raise
+
+            interface.parent_id = related_obj.id
+            interface.save()
+            print(
+                f"🧷 Attached interface {interface} on {interface.device} "
+                f"to parent {related_obj}"
+            )
+        else:
+            query = {r_field: related_value, "device": interface.device, "type": related_field}
+            related_obj, rel_obj_created = r_model.objects.get_or_create(**query)
+
+            if rel_obj_created:
+                setattr(interface, f"{related_field}_id", related_obj.id)
+                interface.save()
+                print(f"🧷 Created {related_field} interface {interface} on {interface.device}")

startup_scripts/220_platforms.py

@@ -1,7 +1,7 @@
 import sys
 
 from dcim.models import Manufacturer, Platform
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 platforms = load_yaml("/opt/netbox/initializers/platforms.yml")
 
@@ -21,7 +21,8 @@ for params in platforms:
 
             params[assoc] = model.objects.get(**query)
 
-    platform, created = Platform.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    platform, created = Platform.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("💾 Created platform", platform.name)

startup_scripts/230_route_targets.py

@@ -1,7 +1,12 @@
 import sys
 
 from ipam.models import RouteTarget
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 
 route_targets = load_yaml("/opt/netbox/initializers/route_targets.yml")
@@ -21,9 +26,10 @@ for params in route_targets:
 
             params[assoc] = model.objects.get(**query)
 
-    route_target, created = RouteTarget.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    route_target, created = RouteTarget.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(route_target, custom_field_data)
-
         print("🎯 Created Route Target", route_target.name)
+
+    set_custom_fields_values(route_target, custom_field_data)

startup_scripts/240_vrfs.py

@@ -1,7 +1,12 @@
 import sys
 
 from ipam.models import VRF
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 
 vrfs = load_yaml("/opt/netbox/initializers/vrfs.yml")
@@ -9,6 +14,7 @@ vrfs = load_yaml("/opt/netbox/initializers/vrfs.yml")
 if vrfs is None:
     sys.exit()
 
+match_params = ["name", "rd"]
 optional_assocs = {"tenant": (Tenant, "name")}
 
 for params in vrfs:
@@ -21,9 +27,10 @@ for params in vrfs:
 
             params[assoc] = model.objects.get(**query)
 
-    vrf, created = VRF.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    vrf, created = VRF.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(vrf, custom_field_data)
-
         print("📦 Created VRF", vrf.name)
+
+    set_custom_fields_values(vrf, custom_field_data)

startup_scripts/250_dcim_interfaces.py

@@ -1,27 +0,0 @@
-import sys
-
-from dcim.models import Device, Interface
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
-
-interfaces = load_yaml("/opt/netbox/initializers/dcim_interfaces.yml")
-
-if interfaces is None:
-    sys.exit()
-
-required_assocs = {"device": (Device, "name")}
-
-for params in interfaces:
-    custom_field_data = pop_custom_fields(params)
-
-    for assoc, details in required_assocs.items():
-        model, field = details
-        query = {field: params.pop(assoc)}
-
-        params[assoc] = model.objects.get(**query)
-
-    interface, created = Interface.objects.get_or_create(**params)
-
-    if created:
-        set_custom_fields_values(interface, custom_field_data)
-
-        print("🧷 Created interface", interface.name, interface.device.name)

startup_scripts/250_rirs.py

@@ -1,7 +1,7 @@
 import sys
 
 from ipam.models import RIR
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 rirs = load_yaml("/opt/netbox/initializers/rirs.yml")
 
@@ -9,7 +9,8 @@ if rirs is None:
     sys.exit()
 
 for params in rirs:
-    rir, created = RIR.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    rir, created = RIR.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🗺️ Created RIR", rir.name)

startup_scripts/260_asns.py

@@ -0,0 +1,34 @@
+import sys
+
+from ipam.models import ASN, RIR
+from startup_script_utils import load_yaml, split_params
+from tenancy.models import Tenant
+
+asns = load_yaml("/opt/netbox/initializers/asns.yml")
+
+if asns is None:
+    sys.exit()
+
+match_params = ["asn", "rir"]
+required_assocs = {"rir": (RIR, "name")}
+optional_assocs = {"tenant": (Tenant, "name")}
+
+for params in asns:
+    for assoc, details in required_assocs.items():
+        model, field = details
+        query = {field: params.pop(assoc)}
+
+        params[assoc] = model.objects.get(**query)
+
+    for assoc, details in optional_assocs.items():
+        if assoc in params:
+            model, field = details
+            query = {field: params.pop(assoc)}
+
+            params[assoc] = model.objects.get(**query)
+
+    matching_params, defaults = split_params(params, match_params)
+    asn, created = ASN.objects.get_or_create(**matching_params, defaults=defaults)
+
+    if created:
+        print(f"🔡 Created ASN {asn.asn}")

startup_scripts/270_aggregates.py

@@ -2,7 +2,12 @@ import sys
 
 from ipam.models import RIR, Aggregate
 from netaddr import IPNetwork
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 
 aggregates = load_yaml("/opt/netbox/initializers/aggregates.yml")
@@ -10,8 +15,8 @@ aggregates = load_yaml("/opt/netbox/initializers/aggregates.yml")
 if aggregates is None:
     sys.exit()
 
+match_params = ["prefix", "rir"]
 required_assocs = {"rir": (RIR, "name")}
-
 optional_assocs = {
     "tenant": (Tenant, "name"),
 }
@@ -34,9 +39,10 @@ for params in aggregates:
 
             params[assoc] = model.objects.get(**query)
 
-    aggregate, created = Aggregate.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    aggregate, created = Aggregate.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(aggregate, custom_field_data)
-
         print("🗞️ Created Aggregate", aggregate.prefix)
+
+    set_custom_fields_values(aggregate, custom_field_data)

startup_scripts/280_prefix_vlan_roles.py

@@ -1,7 +1,7 @@
 import sys
 
 from ipam.models import Role
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 
 roles = load_yaml("/opt/netbox/initializers/prefix_vlan_roles.yml")
 
@@ -9,7 +9,8 @@ if roles is None:
     sys.exit()
 
 for params in roles:
-    role, created = Role.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    role, created = Role.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("⛹️‍ Created Prefix/VLAN Role", role.name)

startup_scripts/280_providers.py

@@ -1,19 +0,0 @@
-import sys
-
-from circuits.models import Provider
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
-
-providers = load_yaml("/opt/netbox/initializers/providers.yml")
-
-if providers is None:
-    sys.exit()
-
-for params in providers:
-    custom_field_data = pop_custom_fields(params)
-
-    provider, created = Provider.objects.get_or_create(**params)
-
-    if created:
-        set_custom_fields_values(provider, custom_field_data)
-
-        print("📡 Created provider", provider.name)

startup_scripts/290_cluster_types.py

@@ -1,6 +1,6 @@
 import sys
 
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from virtualization.models import ClusterType
 
 cluster_types = load_yaml("/opt/netbox/initializers/cluster_types.yml")
@@ -9,7 +9,8 @@ if cluster_types is None:
     sys.exit()
 
 for params in cluster_types:
-    cluster_type, created = ClusterType.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    cluster_type, created = ClusterType.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
         print("🧰 Created Cluster Type", cluster_type.name)

startup_scripts/300_cluster_groups.py

@@ -1,6 +1,6 @@
 import sys
 
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from virtualization.models import ClusterGroup
 
 cluster_groups = load_yaml("/opt/netbox/initializers/cluster_groups.yml")
@@ -9,7 +9,10 @@ if cluster_groups is None:
     sys.exit()
 
 for params in cluster_groups:
-    cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    cluster_group, created = ClusterGroup.objects.get_or_create(
+        **matching_params, defaults=defaults
+    )
 
     if created:
         print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/310_clusters.py

@@ -1,7 +1,12 @@
 import sys
 
 from dcim.models import Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 from virtualization.models import Cluster, ClusterGroup, ClusterType
 
@@ -10,8 +15,8 @@ clusters = load_yaml("/opt/netbox/initializers/clusters.yml")
 if clusters is None:
     sys.exit()
 
+match_params = ["name", "type"]
 required_assocs = {"type": (ClusterType, "name")}
-
 optional_assocs = {
     "site": (Site, "name"),
     "group": (ClusterGroup, "name"),
@@ -34,9 +39,10 @@ for params in clusters:
 
             params[assoc] = model.objects.get(**query)
 
-    cluster, created = Cluster.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    cluster, created = Cluster.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(cluster, custom_field_data)
-
         print("🗄️ Created cluster", cluster.name)
+
+    set_custom_fields_values(cluster, custom_field_data)

startup_scripts/320_vlan_groups.py

@@ -2,7 +2,12 @@ import sys
 
 from django.contrib.contenttypes.models import ContentType
 from ipam.models import VLANGroup
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 
 vlan_groups = load_yaml("/opt/netbox/initializers/vlan_groups.yml")
 
@@ -32,9 +37,11 @@ for params in vlan_groups:
                 )
                 continue
             params["scope_id"] = ct.model_class().objects.get(**query).id
-    vlan_group, created = VLANGroup.objects.get_or_create(**params)
+
+    matching_params, defaults = split_params(params)
+    vlan_group, created = VLANGroup.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(vlan_group, custom_field_data)
-
         print("🏘️ Created VLAN Group", vlan_group.name)
+
+    set_custom_fields_values(vlan_group, custom_field_data)

startup_scripts/330_vlans.py

@@ -2,7 +2,12 @@ import sys
 
 from dcim.models import Site
 from ipam.models import VLAN, Role, VLANGroup
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant, TenantGroup
 
 vlans = load_yaml("/opt/netbox/initializers/vlans.yml")
@@ -10,6 +15,7 @@ vlans = load_yaml("/opt/netbox/initializers/vlans.yml")
 if vlans is None:
     sys.exit()
 
+match_params = ["name", "vid"]
 optional_assocs = {
     "site": (Site, "name"),
     "tenant": (Tenant, "name"),
@@ -28,9 +34,10 @@ for params in vlans:
 
             params[assoc] = model.objects.get(**query)
 
-    vlan, created = VLAN.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    vlan, created = VLAN.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(vlan, custom_field_data)
-
         print("🏠 Created VLAN", vlan.name)
+
+    set_custom_fields_values(vlan, custom_field_data)

startup_scripts/340_virtual_machines.py

@@ -1,7 +1,12 @@
 import sys
 
 from dcim.models import DeviceRole, Platform
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 from virtualization.models import Cluster, VirtualMachine
 
@@ -10,8 +15,8 @@ virtual_machines = load_yaml("/opt/netbox/initializers/virtual_machines.yml")
 if virtual_machines is None:
     sys.exit()
 
+match_params = ["cluster", "name"]
 required_assocs = {"cluster": (Cluster, "name")}
-
 optional_assocs = {
     "tenant": (Tenant, "name"),
     "platform": (Platform, "name"),
@@ -38,9 +43,12 @@ for params in virtual_machines:
 
             params[assoc] = model.objects.get(**query)
 
-    virtual_machine, created = VirtualMachine.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    virtual_machine, created = VirtualMachine.objects.get_or_create(
+        **matching_params, defaults=defaults
+    )
 
     if created:
-        set_custom_fields_values(virtual_machine, custom_field_data)
-
         print("🖥️ Created virtual machine", virtual_machine.name)
+
+    set_custom_fields_values(virtual_machine, custom_field_data)

startup_scripts/350_virtualization_interfaces.py

@@ -1,6 +1,11 @@
 import sys
 
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from virtualization.models import VirtualMachine, VMInterface
 
 interfaces = load_yaml("/opt/netbox/initializers/virtualization_interfaces.yml")
@@ -8,6 +13,7 @@ interfaces = load_yaml("/opt/netbox/initializers/virtualization_interfaces.yml")
 if interfaces is None:
     sys.exit()
 
+match_params = ["name", "virtual_machine"]
 required_assocs = {"virtual_machine": (VirtualMachine, "name")}
 
 for params in interfaces:
@@ -19,9 +25,10 @@ for params in interfaces:
 
         params[assoc] = model.objects.get(**query)
 
-    interface, created = VMInterface.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    interface, created = VMInterface.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(interface, custom_field_data)
-
         print("🧷 Created interface", interface.name, interface.virtual_machine.name)
+
+    set_custom_fields_values(interface, custom_field_data)

startup_scripts/360_prefixes.py

@@ -3,7 +3,12 @@ import sys
 from dcim.models import Site
 from ipam.models import VLAN, VRF, Prefix, Role
 from netaddr import IPNetwork
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant, TenantGroup
 
 prefixes = load_yaml("/opt/netbox/initializers/prefixes.yml")
@@ -11,6 +16,7 @@ prefixes = load_yaml("/opt/netbox/initializers/prefixes.yml")
 if prefixes is None:
     sys.exit()
 
+match_params = ["prefix", "site", "vrf", "vlan"]
 optional_assocs = {
     "site": (Site, "name"),
     "tenant": (Tenant, "name"),
@@ -31,9 +37,10 @@ for params in prefixes:
             query = {field: params.pop(assoc)}
             params[assoc] = model.objects.get(**query)
 
-    prefix, created = Prefix.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    prefix, created = Prefix.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(prefix, custom_field_data)
-
         print("📌 Created Prefix", prefix.prefix)
+
+    set_custom_fields_values(prefix, custom_field_data)

startup_scripts/370_ip_addresses.py

@@ -5,7 +5,12 @@ from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
 from ipam.models import VRF, IPAddress
 from netaddr import IPNetwork
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 from virtualization.models import VirtualMachine, VMInterface
 
@@ -14,10 +19,11 @@ ip_addresses = load_yaml("/opt/netbox/initializers/ip_addresses.yml")
 if ip_addresses is None:
     sys.exit()
 
+match_params = ["address", "vrf"]
 optional_assocs = {
     "tenant": (Tenant, "name"),
     "vrf": (VRF, "name"),
-    "interface": (None, None),
+    "interface": (Interface, "name"),
 }
 
 vm_interface_ct = ContentType.objects.filter(
@@ -52,11 +58,13 @@ for params in ip_addresses:
                     params["assigned_object_id"] = Interface.objects.get(**query).id
             else:
                 query = {field: params.pop(assoc)}
+
                 params[assoc] = model.objects.get(**query)
 
-    ip_address, created = IPAddress.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    ip_address, created = IPAddress.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(ip_address, custom_field_data)
-
         print("🧬 Created IP Address", ip_address.address)
+
+    set_custom_fields_values(ip_address, custom_field_data)

startup_scripts/400_services.py

@@ -2,7 +2,7 @@ import sys
 
 from dcim.models import Device
 from ipam.models import Service
-from startup_script_utils import load_yaml
+from startup_script_utils import load_yaml, split_params
 from virtualization.models import VirtualMachine
 
 services = load_yaml("/opt/netbox/initializers/services.yml")
@@ -10,6 +10,7 @@ services = load_yaml("/opt/netbox/initializers/services.yml")
 if services is None:
     sys.exit()
 
+match_params = ["name", "device", "virtual_machine"]
 optional_assocs = {
     "device": (Device, "name"),
     "virtual_machine": (VirtualMachine, "name"),
@@ -24,6 +25,7 @@ for params in services:
 
             params[assoc] = model.objects.get(**query)
 
+    matching_params, defaults = split_params(params, match_params)
     service, created = Service.objects.get_or_create(**params)
 
     if created:

startup_scripts/420_providers.py

@@ -0,0 +1,25 @@
+import sys
+
+from circuits.models import Provider
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
+
+providers = load_yaml("/opt/netbox/initializers/providers.yml")
+
+if providers is None:
+    sys.exit()
+
+for params in providers:
+    custom_field_data = pop_custom_fields(params)
+
+    matching_params, defaults = split_params(params)
+    provider, created = Provider.objects.get_or_create(**matching_params, defaults=defaults)
+
+    if created:
+        print("📡 Created provider", provider.name)
+
+    set_custom_fields_values(provider, custom_field_data)

startup_scripts/440_circuit_types.py

@@ -1,7 +1,12 @@
 import sys
 
 from circuits.models import CircuitType
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 
 circuit_types = load_yaml("/opt/netbox/initializers/circuit_types.yml")
 
@@ -11,9 +16,10 @@ if circuit_types is None:
 for params in circuit_types:
     custom_field_data = pop_custom_fields(params)
 
-    circuit_type, created = CircuitType.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params)
+    circuit_type, created = CircuitType.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(circuit_type, custom_field_data)
-
         print("⚡ Created Circuit Type", circuit_type.name)
+
+    set_custom_fields_values(circuit_type, custom_field_data)

startup_scripts/450_circuits.py

@@ -1,7 +1,12 @@
 import sys
 
 from circuits.models import Circuit, CircuitType, Provider
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
 from tenancy.models import Tenant
 
 circuits = load_yaml("/opt/netbox/initializers/circuits.yml")
@@ -9,8 +14,8 @@ circuits = load_yaml("/opt/netbox/initializers/circuits.yml")
 if circuits is None:
     sys.exit()
 
+match_params = ["cid", "provider", "type"]
 required_assocs = {"provider": (Provider, "name"), "type": (CircuitType, "name")}
-
 optional_assocs = {"tenant": (Tenant, "name")}
 
 for params in circuits:
@@ -29,9 +34,10 @@ for params in circuits:
 
             params[assoc] = model.objects.get(**query)
 
-    circuit, created = Circuit.objects.get_or_create(**params)
+    matching_params, defaults = split_params(params, match_params)
+    circuit, created = Circuit.objects.get_or_create(**matching_params, defaults=defaults)
 
     if created:
-        set_custom_fields_values(circuit, custom_field_data)
-
         print("⚡ Created Circuit", circuit.cid)
+
+    set_custom_fields_values(circuit, custom_field_data)

startup_scripts/460_cables.py

@@ -0,0 +1,227 @@
+import sys
+from typing import Tuple
+
+from circuits.models import Circuit, CircuitTermination, ProviderNetwork
+from dcim.models import (
+    Cable,
+    ConsolePort,
+    ConsoleServerPort,
+    FrontPort,
+    Interface,
+    PowerFeed,
+    PowerOutlet,
+    PowerPanel,
+    PowerPort,
+    RearPort,
+    Site,
+)
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import Q
+from startup_script_utils import load_yaml
+
+CONSOLE_PORT_TERMINATION = ContentType.objects.get_for_model(ConsolePort)
+CONSOLE_SERVER_PORT_TERMINATION = ContentType.objects.get_for_model(ConsoleServerPort)
+FRONT_PORT_TERMINATION = ContentType.objects.get_for_model(FrontPort)
+REAR_PORT_TERMINATION = ContentType.objects.get_for_model(RearPort)
+FRONT_AND_REAR = [FRONT_PORT_TERMINATION, REAR_PORT_TERMINATION]
+POWER_PORT_TERMINATION = ContentType.objects.get_for_model(PowerPort)
+POWER_OUTLET_TERMINATION = ContentType.objects.get_for_model(PowerOutlet)
+POWER_FEED_TERMINATION = ContentType.objects.get_for_model(PowerFeed)
+POWER_TERMINATIONS = [POWER_PORT_TERMINATION, POWER_OUTLET_TERMINATION, POWER_FEED_TERMINATION]
+
+VIRTUAL_INTERFACES = ["bridge", "lag", "virtual"]
+
+
+def get_termination_object(params: dict, side: str):
+    klass = params.pop(f"termination_{side}_class")
+    name = params.pop(f"termination_{side}_name", None)
+    device = params.pop(f"termination_{side}_device", None)
+    feed_params = params.pop(f"termination_{side}_feed", None)
+    circuit_params = params.pop(f"termination_{side}_circuit", {})
+
+    if name and device:
+        termination = klass.objects.get(name=name, device__name=device)
+        return termination
+    elif feed_params:
+        q = {
+            "name": feed_params["power_panel"]["name"],
+            "site__name": feed_params["power_panel"]["site"],
+        }
+        power_panel = PowerPanel.objects.get(**q)
+        termination = PowerFeed.objects.get(name=feed_params["name"], power_panel=power_panel)
+        return termination
+    elif circuit_params:
+        circuit = Circuit.objects.get(cid=circuit_params.pop("cid"))
+        term_side = circuit_params.pop("term_side").upper()
+
+        site_name = circuit_params.pop("site", None)
+        provider_network = circuit_params.pop("provider_network", None)
+
+        if site_name:
+            circuit_params["site"] = Site.objects.get(name=site_name)
+        elif provider_network:
+            circuit_params["provider_network"] = ProviderNetwork.objects.get(name=provider_network)
+        else:
+            raise ValueError(
+                f"⚠️ Missing one of required parameters: 'site' or 'provider_network' "
+                f"for side {term_side} of circuit {circuit}"
+            )
+
+        termination, created = CircuitTermination.objects.get_or_create(
+            circuit=circuit, term_side=term_side, defaults=circuit_params
+        )
+        if created:
+            print(f"⚡ Created new CircuitTermination {termination}")
+
+        return termination
+
+    raise ValueError(
+        f"⚠️ Missing parameters for termination_{side}. "
+        "Need termination_{side}_name AND termination_{side}_device OR termination_{side}_circuit"
+    )
+
+
+def get_termination_class_by_name(port_class: str):
+    if not port_class:
+        return Interface
+
+    return globals()[port_class]
+
+
+def cable_in_cables(term_a: tuple, term_b: tuple) -> bool:
+    """Check if cable exist for given terminations.
+    Each tuple should consist termination object and termination type
+    """
+
+    cable = Cable.objects.filter(
+        Q(
+            termination_a_id=term_a[0].id,
+            termination_a_type=term_a[1],
+            termination_b_id=term_b[0].id,
+            termination_b_type=term_b[1],
+        )
+        | Q(
+            termination_a_id=term_b[0].id,
+            termination_a_type=term_b[1],
+            termination_b_id=term_a[0].id,
+            termination_b_type=term_a[1],
+        )
+    )
+    return cable.exists()
+
+
+def check_termination_types(type_a, type_b) -> Tuple[bool, str]:
+    if type_a in POWER_TERMINATIONS and type_b in POWER_TERMINATIONS:
+        if type_a == type_b:
+            return False, "Can't connect the same power terminations together"
+        elif (
+            type_a == POWER_OUTLET_TERMINATION
+            and type_b == POWER_FEED_TERMINATION
+            or type_a == POWER_FEED_TERMINATION
+            and type_b == POWER_OUTLET_TERMINATION
+        ):
+            return False, "PowerOutlet can't be connected with PowerFeed"
+    elif type_a in POWER_TERMINATIONS or type_b in POWER_TERMINATIONS:
+        return False, "Can't mix power terminations with port terminations"
+    elif type_a in FRONT_AND_REAR or type_b in FRONT_AND_REAR:
+        return True, ""
+    elif (
+        type_a == CONSOLE_PORT_TERMINATION
+        and type_b != CONSOLE_SERVER_PORT_TERMINATION
+        or type_b == CONSOLE_PORT_TERMINATION
+        and type_a != CONSOLE_SERVER_PORT_TERMINATION
+    ):
+        return False, "ConsolePorts can only be connected to ConsoleServerPorts or Front/Rear ports"
+    return True, ""
+
+
+def get_cable_name(termination_a: tuple, termination_b: tuple) -> str:
+    """Returns name of a cable in format:
+    device_a interface_a <---> interface_b device_b
+    or for circuits:
+    circuit_a termination_a <---> termination_b circuit_b
+    """
+    cable_name = []
+
+    for is_side_b, termination in enumerate([termination_a, termination_b]):
+        try:
+            power_panel_id = getattr(termination[0], "power_panel_id", None)
+            if power_panel_id:
+                power_feed = PowerPanel.objects.get(id=power_panel_id)
+                segment = [f"{power_feed}", f"{termination[0]}"]
+            else:
+                segment = [f"{termination[0].device}", f"{termination[0]}"]
+        except AttributeError:
+            segment = [f"{termination[0].circuit.cid}", f"{termination[0]}"]
+
+        if is_side_b:
+            segment.reverse()
+
+        cable_name.append(" ".join(segment))
+
+    return " <---> ".join(cable_name)
+
+
+def check_interface_types(*args):
+    for termination in args:
+        try:
+            if termination.type in VIRTUAL_INTERFACES:
+                raise Exception(
+                    f"⚠️ Virtual interfaces are not supported for cabling. "
+                    f"Termination {termination.device} {termination} {termination.type}"
+                )
+        except AttributeError:
+            # CircuitTermination doesn't have a type field
+            pass
+
+
+def check_terminations_are_free(*args):
+    any_failed = False
+    for termination in args:
+        if termination.cable_id:
+            any_failed = True
+            print(
+                f"⚠️ Termination {termination} is already occupied "
+                f"with cable #{termination.cable_id}"
+            )
+    if any_failed:
+        raise Exception("⚠️ At least one end of the cable is already occupied.")
+
+
+cables = load_yaml("/opt/netbox/initializers/cables.yml")
+
+if cables is None:
+    sys.exit()
+
+for params in cables:
+    params["termination_a_class"] = get_termination_class_by_name(params.get("termination_a_class"))
+    params["termination_b_class"] = get_termination_class_by_name(params.get("termination_b_class"))
+
+    term_a = get_termination_object(params, side="a")
+    term_b = get_termination_object(params, side="b")
+
+    check_interface_types(term_a, term_b)
+
+    term_a_ct = ContentType.objects.get_for_model(term_a)
+    term_b_ct = ContentType.objects.get_for_model(term_b)
+
+    types_ok, msg = check_termination_types(term_a_ct, term_b_ct)
+    cable_name = get_cable_name((term_a, term_a_ct), (term_b, term_b_ct))
+
+    if not types_ok:
+        print(f"⚠️ Invalid termination types for {cable_name}. {msg}")
+        continue
+
+    if cable_in_cables((term_a, term_a_ct), (term_b, term_b_ct)):
+        continue
+
+    check_terminations_are_free(term_a, term_b)
+
+    params["termination_a_id"] = term_a.id
+    params["termination_b_id"] = term_b.id
+    params["termination_a_type"] = term_a_ct
+    params["termination_b_type"] = term_b_ct
+
+    cable = Cable.objects.create(**params)
+
+    print(f"🧷 Created cable {cable} {cable_name}")

startup_scripts/470_contact_groups.py

@@ -0,0 +1,36 @@
+import sys
+
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
+from tenancy.models import ContactGroup
+
+contact_groups = load_yaml("/opt/netbox/initializers/contact_groups.yml")
+
+if contact_groups is None:
+    sys.exit()
+
+optional_assocs = {"parent": (ContactGroup, "name")}
+
+for params in contact_groups:
+    custom_field_data = pop_custom_fields(params)
+
+    for assoc, details in optional_assocs.items():
+        if assoc in params:
+            model, field = details
+            query = {field: params.pop(assoc)}
+
+            params[assoc] = model.objects.get(**query)
+
+    matching_params, defaults = split_params(params)
+    contact_group, created = ContactGroup.objects.get_or_create(
+        **matching_params, defaults=defaults
+    )
+
+    if created:
+        print("🔳 Created Contact Group", contact_group.name)
+
+    set_custom_fields_values(contact_group, custom_field_data)

startup_scripts/480_contact_roles.py

@@ -0,0 +1,26 @@
+import sys
+
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
+from tenancy.models import ContactRole
+
+contact_roles = load_yaml("/opt/netbox/initializers/contact_roles.yml")
+
+if contact_roles is None:
+    sys.exit()
+
+
+for params in contact_roles:
+    custom_field_data = pop_custom_fields(params)
+
+    matching_params, defaults = split_params(params)
+    contact_role, created = ContactRole.objects.get_or_create(**matching_params, defaults=defaults)
+
+    if created:
+        print("🔳 Created Contact Role", contact_role.name)
+
+    set_custom_fields_values(contact_role, custom_field_data)

startup_scripts/490_contacts.py

@@ -0,0 +1,34 @@
+import sys
+
+from startup_script_utils import (
+    load_yaml,
+    pop_custom_fields,
+    set_custom_fields_values,
+    split_params,
+)
+from tenancy.models import Contact, ContactGroup
+
+contacts = load_yaml("/opt/netbox/initializers/contacts.yml")
+
+if contacts is None:
+    sys.exit()
+
+optional_assocs = {"group": (ContactGroup, "name")}
+
+for params in contacts:
+    custom_field_data = pop_custom_fields(params)
+
+    for assoc, details in optional_assocs.items():
+        if assoc in params:
+            model, field = details
+            query = {field: params.pop(assoc)}
+
+            params[assoc] = model.objects.get(**query)
+
+    matching_params, defaults = split_params(params)
+    contact, created = Contact.objects.get_or_create(**matching_params, defaults=defaults)
+
+    if created:
+        print("👩‍💻 Created Contact", contact.name)
+
+    set_custom_fields_values(contact, custom_field_data)

startup_scripts/startup_script_utils/__init__.py

@@ -1,2 +1,3 @@
 from .custom_fields import pop_custom_fields, set_custom_fields_values
 from .load_yaml import load_yaml
+from .utils import split_params

startup_scripts/startup_script_utils/custom_fields.py

@@ -1,9 +1,38 @@
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ObjectDoesNotExist
+from extras.models import CustomField
+
+
 def set_custom_fields_values(entity, custom_field_data):
     if not custom_field_data:
         return
 
-    entity.custom_field_data = custom_field_data
-    return entity.save()
+    missing_cfs = []
+    save = False
+    for key, value in custom_field_data.items():
+        try:
+            cf = CustomField.objects.get(name=key)
+        except ObjectDoesNotExist:
+            missing_cfs.append(key)
+        else:
+            ct = ContentType.objects.get_for_model(entity)
+            if ct not in cf.content_types.all():
+                print(
+                    f"⚠️ Custom field {key} is not enabled for {entity}'s model!"
+                    "Please check the 'on_objects' for that custom field in custom_fields.yml"
+                )
+            elif key not in entity.custom_field_data:
+                entity.custom_field_data[key] = value
+                save = True
+
+    if missing_cfs:
+        raise Exception(
+            f"⚠️ Custom field(s) '{missing_cfs}' requested for {entity} but not found in Netbox!"
+            "Please chceck the custom_fields.yml"
+        )
+
+    if save:
+        entity.save()
 
 
 def pop_custom_fields(params):

startup_scripts/startup_script_utils/utils.py

@@ -0,0 +1,15 @@
+from typing import Tuple
+
+
+def split_params(params: dict, unique_params: list = None) -> Tuple[dict, dict]:
+    """Split params dict into dict with matching params and a dict with default values"""
+
+    if unique_params is None:
+        unique_params = ["name", "slug"]
+
+    matching_params = {}
+    for unique_param in unique_params:
+        param = params.pop(unique_param, None)
+        if param:
+            matching_params[unique_param] = param
+    return matching_params, params