Merge branch 'miso231-patch-1'

Thank you Digitalocean for sponsoring Netbox!
And thank you for letting it grow up and move out ;)

Dockerfile

@@ -25,24 +25,38 @@ RUN pip install \
 # pinning django to the version required by netbox
 # adding it here, to install the correct version of
 # django-rq
-      'Django>=2.1.5,<2.2' \
+      'Django>=2.2,<2.3' \
 # django-rq is used for webhooks
       django-rq
 
+ARG BRANCH=master
+
+WORKDIR /tmp
+
+# As the requirements don't change very often,
+# and as they take some time to compile,
+# we try to cache them very agressively.
+ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt
+ADD ${REQUIREMENTS_URL} requirements.txt
+RUN pip install -r requirements.txt
+
+# Cache bust when the upstream branch changes:
+# ADD will fetch the file and check if it has changed
+# If not, Docker will use the existing build cache.
+# If yes, Docker will bust the cache and run every build step from here on.
+ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH
+ADD ${REF_URL} version.json
+
 WORKDIR /opt
 
-ARG BRANCH=master
+ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz
-ARG URL=https://github.com/digitalocean/netbox/archive/$BRANCH.tar.gz
 RUN wget -q -O - "${URL}" | tar xz \
   && mv netbox* netbox
 
-WORKDIR /opt/netbox
-RUN pip install -r requirements.txt
-
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
 COPY configuration/gunicorn_config.py /etc/netbox/config/
 COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf
-COPY docker/docker-entrypoint.sh docker-entrypoint.sh
+COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
 COPY configuration/configuration.py /etc/netbox/config/configuration.py

README.md

@@ -13,17 +13,17 @@ Questions? Before opening an issue on Github, please join the [Network To Code][
 
 To get Netbox up and running:
 
-```
+```bash
-$ git clone -b master https://github.com/netbox-community/netbox-docker.git
+git clone -b master https://github.com/netbox-community/netbox-docker.git
-$ cd netbox-docker
+cd netbox-docker
-$ docker-compose pull
+docker-compose pull
-$ docker-compose up -d
+docker-compose up -d
 ```
 
 The application will be available after a few minutes.
 Use `docker-compose port nginx 8080` to find out where to connect to.
 
-```
+```bash
 $ echo "http://$(docker-compose port nginx 8080)/"
 http://0.0.0.0:32768/
 
@@ -85,12 +85,7 @@ You may run this image in a cluster such as Docker Swarm, Kubernetes or OpenShif
 
 In this case, we encourage you to statically configure Netbox by starting from [Netbox's example config file][default-config], and mounting it into your container in the directory `/etc/netbox/config/` using the mechanism provided by your container platform (i.e. [Docker Swarm configs][swarm-config], [Kubernetes ConfigMap][k8s-config], [OpenShift ConfigMaps][openshift-config]).
 
-But if you rather continue to configure your application through environment variables, you may continue to use [the built-in configuration file][
+But if you rather continue to configure your application through environment variables, you may continue to use [the built-in configuration file][docker-config].
-
-
-
-
-].
 We discourage storing secrets in environment variables, as environment variable are passed on to all sub-processes and may leak easily into other systems, e.g. error collecting tools that often collect all environment variables whenever an error occurs.
 
 Therefore we *strongly advise* to make use of the secrets mechanism provided by your container platform (i.e. [Docker Swarm secrets][swarm-secrets], [Kubernetes secrets][k8s-secrets], [OpenShift secrets][openshift-secrets]).
@@ -104,11 +99,12 @@ If a secret is defined by an environment variable and in the respective file at
 * `EMAIL_PASSWORD`: `/run/secrets/email_password`
 * `NAPALM_PASSWORD`: `/run/secrets/napalm_password`
 * `REDIS_PASSWORD`: `/run/secrets/redis_password`
+* `AUTH_LDAP_BIND_PASSWORD`: `/run/secrets/auth_ldap_bind_password`
 
 Please also consider [the advice about running Netbox in production](#production) above!
 
 [docker-config]: https://github.com/netbox-community/netbox-docker/blob/master/configuration/configuration.py
-[default-config]: https://github.com/digitalocean/netbox/blob/develop/netbox/netbox/configuration.example.py
+[default-config]: https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py
 [entrypoint]: https://github.com/netbox-community/netbox-docker/blob/master/docker/docker-entrypoint.sh
 [swarm-config]: https://docs.docker.com/engine/swarm/configs/
 [swarm-secrets]: https://docs.docker.com/engine/swarm/secrets/
@@ -218,7 +214,7 @@ echo "from django.contrib.auth.models import Permission\nfor p in Permission.obj
 You can also build your own Netbox Docker image containing your own startup scripts, custom fields, users and groups
 like this:
 
-```
+```Dockerfile
 ARG VERSION=latest
 FROM netboxcommunity/netbox:$VERSION
 
@@ -233,27 +229,27 @@ To use this feature, set the environment-variable `VERSION` before launching `do
 `VERSION` may be set to the name of
 [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
 
-```
+```bash
-$ export VERSION=v2.2.6
+export VERSION=v2.2.6
-$ docker-compose pull netbox
+docker-compose pull netbox
-$ docker-compose up -d
+docker-compose up -d
 ```
 
 You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid
-[Git Reference][git-ref] declared on [the 'digitalocean/netbox' Github repository][netbox-github].
+[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github].
 Most commonly you will specify a tag or branch name.
 
-```
+```bash
-$ export VERSION=develop
+export VERSION=develop
-$ docker-compose build --no-cache netbox
+docker-compose build --no-cache netbox
-$ docker-compose up -d
+docker-compose up -d
 ```
 
 Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary.
 This can increase the build speed if you're just adjusting the config, for example.
 
 [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
-[netbox-github]: https://github.com/digitalocean/netbox/releases
+[netbox-github]: https://github.com/netbox-community/netbox/releases
 
 ### LDAP enabled variant
 
@@ -303,7 +299,7 @@ Now start everything up again.
 If this didn't help, try to see if there's anything in the logs indicating why nginx doesn't start:
 
 ```bash
-$ docker-compose logs -f nginx
+docker-compose logs -f nginx
 ```
 
 ### Getting a "Bad Request (400)"
@@ -330,14 +326,14 @@ docker-compose up -d netbox netbox-worker
 First make sure that the webhooks feature is enabled in your Netbox configuration and that a redis host is defined.
 Check `netbox.env` if the following variables are defined:
 
-```
+```bash
 WEBHOOKS_ENABLED=true
 REDIS_HOST=redis
 ```
 
 Then make sure that the `redis` container and at least one `netbox-worker` are running.
 
-```
+```bash
 # check the container status
 $ docker-compose ps
 
@@ -374,7 +370,7 @@ docker-compose run --rm -T redis sh -c 'redis-cli -h redis -a $REDIS_PASSWORD mo
 
 If you don't see anything happening after you triggered a webhook, double-check the configuration of the `netbox` and the `netbox-worker` containers and also check the configuration of your webhook in the admin interface of Netbox.
 
-### Breaking Changes
+## Breaking Changes
 
 From time to time it might become necessary to re-engineer the structure of this setup.
 Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence.
@@ -384,6 +380,22 @@ Compare the version with the list below to check whether a breaking change was i
 
 The following is a list of breaking changes of the `netbox-docker` project:
 
+* 0.16.0: Update the Netbox URL from "github.com/digitalocean/netbox" to "github.com/netbox-community/netbox"
+* 0.15.0: Update for Netbox v2.6.0.
+  The `configuration/configuration.py` file has been updated to match the file from Netbox.
+  `CORS_ORIGIN_WHITELIST` has a new default value of `http://localhost`.
+  To provide a nice development environment, `CORS_ORIGIN_ALLOW_ALL` added to `env/netbox.env` with a default value of `True`.
+  There are also new options:
+  * `REDIS_CACHE_DATABASE`
+  * `CACHE_TIMEOUT` (set to 0 to disable caching)
+  * `CHANGELOG_RETENTION`
+  * `CORS_ORIGIN_REGEX_WHITELIST` (space separated list of regular expressions)
+  * `EXEMPT_VIEW_PERMISSIONS` (space separated list)
+  * `METRICS_ENABLED`
+* 0.14.0: Improved caching strategy [#137][137] [#136][136].
+  New `AUTH_LDAP_GROUP_TYPE` environment variable [#135][135].
+* 0.13.0: `AUTH_LDAP_BIND_PASSWORD` can now be extracted into a secrets file. [#133][133]
+* 0.12.0: A new flag `REDIS_SSL=false` was added to the `env/netbox.env` file. [#129][129]
 * 0.11.0: The docker-compose file now marks volumes as shared (`:z`). This should prevent SELinux problems [#131][131]
 * 0.9.0: Upgrade to at least 2.1.5
 * 0.8.0: Alpine linux was upgraded to 3.9 [#126][126]
@@ -399,6 +411,11 @@ The following is a list of breaking changes of the `netbox-docker` project:
 [54]: https://github.com/netbox-community/netbox-docker/issues/54
 [126]: https://github.com/netbox-community/netbox-docker/pull/126
 [131]: https://github.com/netbox-community/netbox-docker/pull/131
+[129]: https://github.com/netbox-community/netbox-docker/pull/129
+[133]: https://github.com/netbox-community/netbox-docker/pull/133
+[135]: https://github.com/netbox-community/netbox-docker/pull/135
+[136]: https://github.com/netbox-community/netbox-docker/pull/136
+[137]: https://github.com/netbox-community/netbox-docker/pull/137
 
 ## Rebuilding & Publishing images
 
@@ -415,8 +432,8 @@ New Docker images are built and published every 24h on the [Docker Build Infrast
 
 To run the tests coming with Netbox, use the `docker-compose.yml` file as such:
 
-```
+```bash
-$ docker-compose run netbox ./manage.py test
+docker-compose run netbox ./manage.py test
 ```
 
 ## About

VERSION

@@ -1 +1 @@
-0.11.0
+0.15.0

build-branches.sh

@@ -11,7 +11,7 @@ else
   GITHUB_OAUTH_PARAMS=""
 fi
 
-ORIGINAL_GITHUB_REPO="${SRC_ORG-digitalocean}/${SRC_REPO-netbox}"
+ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
 GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
 URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
 

build-latest.sh

@@ -11,7 +11,7 @@ else
   GITHUB_OAUTH_PARAMS=""
 fi
 
-ORIGINAL_GITHUB_REPO="digitalocean/netbox"
+ORIGINAL_GITHUB_REPO="netbox-community/netbox"
 GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
 URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}"
 

build.sh

@@ -38,7 +38,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "           This is used to tag all patch releases to their containing version e.g. v2.5.1 -> v2.5"
   echo "           Default: <DOCKER_ORG>/<DOCKER_REPO>:\$MAJOR.\$MINOR"
   echo "  SRC_ORG  Which fork of netbox to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)."
-  echo "           Default: digitalocean"
+  echo "           Default: netbox-community"
   echo "  SRC_REPO The name of the netbox for to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)."
   echo "           Default: netbox"
   echo "  URL      Where to fetch the package from."
@@ -81,7 +81,7 @@ fi
 NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
 
 # variables for fetching the source
-SRC_ORG="${SRC_ORG-digitalocean}"
+SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
 BRANCH="${1}"
 URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}"

configuration/configuration.py

@@ -1,8 +1,9 @@
 import os
+import re
 import socket
 
 # For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/
-# Based on https://github.com/digitalocean/netbox/blob/develop/netbox/netbox/configuration.example.py
+# Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py
 
 # Read secret from file
 def read_secret(secret_name):
@@ -44,6 +45,17 @@ DATABASE = {
 # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY
 SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key'))
 
+# Redis database settings. The Redis database is used for caching and background processing such as webhooks
+REDIS = {
+    'HOST': os.environ.get('REDIS_HOST', 'localhost'),
+    'PORT': int(os.environ.get('REDIS_PORT', 6379)),
+    'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')),
+    'DATABASE': os.environ.get('REDIS_DATABASE', '0'),
+    'CACHE_DATABASE': os.environ.get('REDIS_CACHE_DATABASE', '1'),
+    'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'),
+    'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true',
+}
+
 #########################
 #                       #
 #   Optional settings   #
@@ -68,14 +80,18 @@ BANNER_LOGIN = os.environ.get('BANNER_LOGIN', '')
 # BASE_PATH = 'netbox/'
 BASE_PATH = os.environ.get('BASE_PATH', '')
 
+# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
+CACHE_TIMEOUT = int(os.environ.get('CACHE_TIMEOUT', 900))
+
+# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
+CHANGELOG_RETENTION = int(os.environ.get('CHANGELOG_RETENTION', 90))
+
 # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be
 # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or
 # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers
 CORS_ORIGIN_ALLOW_ALL = os.environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true'
-CORS_ORIGIN_WHITELIST = os.environ.get('CORS_ORIGIN_WHITELIST', '').split(' ')
+CORS_ORIGIN_WHITELIST = list(filter(None, os.environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' ')))
-CORS_ORIGIN_REGEX_WHITELIST = [
+CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, os.environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))]
-    # r'^(https?://)?(\w+\.)?example\.com$',
-]
 
 # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal
 # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging
@@ -97,6 +113,10 @@ EMAIL = {
 # set ENFORCE_GLOBAL_UNIQUE to True.
 ENFORCE_GLOBAL_UNIQUE = os.environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true'
 
+# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and
+# by anonymous users. List models in the form `<app>.<model>`. Add '*' to this list to exempt all models.
+EXEMPT_VIEW_PERMISSIONS = list(filter(None, os.environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' ')))
+
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
 #   https://docs.djangoproject.com/en/1.11/topics/logging/
 LOGGING = {}
@@ -117,6 +137,9 @@ MAX_PAGE_SIZE = int(os.environ.get('MAX_PAGE_SIZE', 1000))
 # the default value of this setting is derived from the installed location.
 MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media'))
 
+# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics'
+METRICS_ENABLED = os.environ.get('METRICS_ENABLED', 'False').lower() == 'true'
+
 # Credentials that NetBox will use to access live devices.
 NAPALM_USERNAME = os.environ.get('NAPALM_USERNAME', '')
 NAPALM_PASSWORD = os.environ.get('NAPALM_PASSWORD', read_secret('napalm_password'))
@@ -134,20 +157,6 @@ PAGINATE_COUNT = int(os.environ.get('PAGINATE_COUNT', 50))
 # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to
 # prefer IPv4 instead.
 PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true'
-
-# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis
-# database be configured and accessible by NetBox (see `REDIS` below).
-WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true'
-
-# Redis database settings (optional). A Redis database is required only if the webhooks backend is enabled.
-REDIS = {
-    'HOST': os.environ.get('REDIS_HOST', 'localhost'),
-    'PORT': os.environ.get('REDIS_PORT', 6379),
-    'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')),
-    'DATABASE': os.environ.get('REDIS_DATABASE', '0'),
-    'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'),
-}
-
 # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of
 # this setting is derived from the installed location.
 REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports')
@@ -155,6 +164,10 @@ REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports')
 # Time zone (default: UTC)
 TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC')
 
+# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis
+# database be configured and accessible by NetBox (see `REDIS` below).
+WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true'
+
 # Date/time formatting. See the following link for supported formats:
 # https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
 DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y')

configuration/ldap_config.py

@@ -1,7 +1,26 @@
 import ldap
 import os
 
-from django_auth_ldap.config import LDAPSearch, GroupOfNamesType
+from django_auth_ldap.config import LDAPSearch
+from importlib import import_module
+
+# Read secret from file
+def read_secret(secret_name):
+    try:
+        f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
+    except EnvironmentError:
+        return ''
+    else:
+        with f:
+            return f.readline().strip()
+
+# Import and return the group type based on string name
+def import_group_type(group_type_name):
+    mod = import_module('django_auth_ldap.config')
+    try:
+        return getattr(mod, group_type_name)()
+    except:
+        return None
 
 # Server URI
 AUTH_LDAP_SERVER_URI = os.environ.get('AUTH_LDAP_SERVER_URI', '')
@@ -13,7 +32,7 @@ AUTH_LDAP_CONNECTION_OPTIONS = {
 
 # Set the DN and password for the NetBox service account.
 AUTH_LDAP_BIND_DN = os.environ.get('AUTH_LDAP_BIND_DN', '')
-AUTH_LDAP_BIND_PASSWORD = os.environ.get('AUTH_LDAP_BIND_PASSWORD', '')
+AUTH_LDAP_BIND_PASSWORD = os.environ.get('AUTH_LDAP_BIND_PASSWORD', read_secret('auth_ldap_bind_password'))
 
 # Set a string template that describes any user’s distinguished name based on the username.
 AUTH_LDAP_USER_DN_TEMPLATE = os.environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None)
@@ -35,7 +54,7 @@ AUTH_LDAP_GROUP_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN',
 AUTH_LDAP_GROUP_SEARCH_CLASS = os.environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group')
 AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE,
                                     "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")")
-AUTH_LDAP_GROUP_TYPE = GroupOfNamesType()
+AUTH_LDAP_GROUP_TYPE = import_group_type(os.environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType'))
 
 # Define a group required to login.
 AUTH_LDAP_REQUIRE_GROUP = os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '')

env/netbox.env

@@ -1,3 +1,4 @@
+CORS_ORIGIN_ALLOW_ALL=True
 DB_NAME=netbox
 DB_USER=netbox
 DB_PASSWORD=J5brHrAXFLQSif0K
@@ -15,6 +16,9 @@ NAPALM_TIMEOUT=10
 MAX_PAGE_SIZE=1000
 REDIS_HOST=redis
 REDIS_PASSWORD=H733Kdjndks81
+REDIS_DATABASE=0
+REDIS_CACHE_DATABASE=1
+REDIS_SSL=false
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 SUPERUSER_NAME=admin
 SUPERUSER_EMAIL=admin@example.com