diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4932ae8..d803cbc 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,8 @@ jobs: build_cmd: - ./build-latest.sh - PRERELEASE=true ./build-latest.sh - - ./build-branches.sh + - ./build-next.sh + - ./build.sh develop docker_from: - '' # use the default of the DOCKERFILE - python:3.7-alpine diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1796097..9dbf770 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,14 +14,15 @@ jobs: build_cmd: - ./build-latest.sh - PRERELEASE=true ./build-latest.sh - - ./build-branches.sh + - ./build-next.sh + - ./build.sh develop fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images steps: - id: git-checkout name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v2 - id: docker-build name: Build the image with '${{ matrix.build_cmd }}' run: ${{ matrix.build_cmd }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 99c5b53..0000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -sudo: required -language: python - -env: - - BUILD=release - - BUILD=prerelease - - BUILD=branches - - BUILD=special - -git: - depth: 5 - -services: - - docker - -install: - - docker-compose pull --parallel - - docker-compose build - -script: - - docker-compose run netbox ./manage.py test - -after_script: - - docker-compose down - -after_success: - - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - - ./build-all.sh --push - -notifications: - slack: - secure: F3VsWcvU/XYyjGjU8ZAVGpREe7F1NjKq6LuMRzhQORbXUvanxDQtLzEe0Y5zm/6+gHkx6t8cX/v2PiCI+v46pkapYMUimd+QEOL1WxbUdnw2kQxcgw/R3wX34l2FHXbG3/a+TmH3euqbSCTIrPy9ufju948i+Q0E0u0fyInmozl8qOT23C4joQOpVAq7y+wHxTxsEg46ZzL2Ties+dmqjMsvHocv7mPI2IWzAWA8SJZxS82Amoapww++QjgEmoY+tMimLkdeXCRgeoj41UGHDg54rbEXh/PTaWiuzyzTr1WLmsGRScC57fDRivp3mSM37/MlNxsRj1z+j4zrvWFQgNfJ2yMjBHroc1jOX/uCY4dwbpSPqUCpc4idMGCGZFItgzTQ3lAPYAsom0C6n8C08Xk8EsNKWwXrDSd4ZUhIwptkNPCFK+kXbLFsMzSApnaBYW0T+wba57nZdiWjOPYmvJr49MDm5NHv2KaRBX2gpw7t7ZLhTgwGEWcZvcDebiLneXcXY5hZ7v2NHJkx/2x1yNXo85xZDy0wK1FGoOOHwPhvqOB+pcQZ/pUOSPTKqGw5l/CexoRm1shFsK+19FnSgimqTHjcuCo4lFW3JlEvlFhtfFXIte2Wjp1ALZgTrSq8zSD5rRxYCUKmM7b3EJwdaIgbvKWPdS4sCXlXU1bHx0g= diff --git a/Dockerfile b/Dockerfile index 2c83172..4347aa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=python:3.7-alpine +ARG FROM FROM ${FROM} as builder RUN apk add --no-cache \ diff --git a/build-all.sh b/build-all.sh deleted file mode 100755 index 462a83a..0000000 --- a/build-all.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Builds all Docker images this project provides -# Arguments: -# BUILD: The release to build. -# Allowed: release, prerelease, branches, special -# Default: undefined - -echo "▶️ $0 $*" - -ALL_BUILDS=("release" "prerelease" "branches" "special") -BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}") - -echo "⚙️ Configured builds: ${BUILDS[*]}" - -if [ -n "${DEBUG}" ]; then - export DEBUG -fi - -ERROR=0 - -for BUILD in "${BUILDS[@]}"; do - echo "🛠 Building '$BUILD' from '$DOCKERFILE'" - case $BUILD in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ || ERROR=1 - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ || ERROR=1 - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ || ERROR=1 - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac -done - -exit $ERROR diff --git a/build-functions/docker-functions.sh b/build-functions/docker-functions.sh new file mode 100644 index 0000000..137ec54 --- /dev/null +++ b/build-functions/docker-functions.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +push_image_to_registry() { + local target_tag=$1 + echo "⏫ Pushing '${target_tag}'" + $DRY docker push "${target_tag}" + echo "✅ Finished pushing the Docker image '${target_tag}'." +} \ No newline at end of file diff --git a/build-functions/get-public-image-config.sh b/build-functions/get-public-image-config.sh new file mode 100644 index 0000000..f718716 --- /dev/null +++ b/build-functions/get-public-image-config.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Retrieves image configuration from public images in DockerHub +# Functions from https://gist.github.com/cirocosta/17ea17be7ac11594cb0f290b0a3ac0d1 +# Optimised for our use case + +get_image_label() { + local label=$1 + local image=$2 + local tag=$3 + local token + token=$(_get_token "$image") + local digest + digest=$(_get_digest "$image" "$tag" "$token") + local retval="null" + if [ "$digest" != "null" ]; then + retval=$(_get_image_configuration "$image" "$token" "$digest" "$label") + fi + echo "$retval" +} + +get_image_layers() { + local image=$1 + local tag=$2 + local token + token=$(_get_token "$image") + _get_layers "$image" "$tag" "$token" +} + +get_image_last_layer() { + local image=$1 + local tag=$2 + local token + token=$(_get_token "$image") + local layers + mapfile -t layers < <(_get_layers "$image" "$tag" "$token") + echo "${layers[-1]}" +} + +_get_image_configuration() { + local image=$1 + local token=$2 + local digest=$3 + local label=$4 + curl \ + --silent \ + --location \ + --header "Authorization: Bearer $token" \ + "https://registry-1.docker.io/v2/$image/blobs/$digest" \ + | jq -r ".config.Labels.\"$label\"" +} + +_get_token() { + local image=$1 + curl \ + --silent \ + "https://auth.docker.io/token?scope=repository:$image:pull&service=registry.docker.io" \ + | jq -r '.token' +} + +_get_digest() { + local image=$1 + local tag=$2 + local token=$3 + curl \ + --silent \ + --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + --header "Authorization: Bearer $token" \ + "https://registry-1.docker.io/v2/$image/manifests/$tag" \ + | jq -r '.config.digest' +} + +_get_layers() { + local image=$1 + local tag=$2 + local token=$3 + curl \ + --silent \ + --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + --header "Authorization: Bearer $token" \ + "https://registry-1.docker.io/v2/$image/manifests/$tag" \ + | jq -r '.layers[].digest' +} diff --git a/build-branches.sh b/build-next.sh similarity index 72% rename from build-branches.sh rename to build-next.sh index 483e771..f134333 100755 --- a/build-branches.sh +++ b/build-next.sh @@ -23,25 +23,17 @@ GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}" # Composing the JQ commans to extract the most recent version number -JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string' +JQ_NEXT='map(.name) | .[] | scan("^[^v].+") | match("^(develop-).*") | .string' CURL="curl -sS" # Querying the Github API to fetch all branches -BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES") +NEXT=$($CURL "${URL_RELEASES}" | jq -r "$JQ_NEXT") -### -# Building each branch -### - -# keeping track whether an error occured -ERROR=0 - -# calling build.sh for each branch -for BRANCH in $BRANCHES; do +if [ -n "$NEXT" ]; then # shellcheck disable=SC2068 - ./build.sh "${BRANCH}" $@ || ERROR=1 -done - -# returning whether an error occured -exit $ERROR + ./build.sh "${NEXT}" $@ +else + echo "No branch matching 'develop-*' found" + echo "::set-output name=skipped::true" +fi diff --git a/build.sh b/build.sh index 051cffb..7489ebb 100755 --- a/build.sh +++ b/build.sh @@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" echo " DOCKER_FROM The base image to use." - echo " Default: Whatever is defined as default in the Dockerfile." + echo " Default: 'python:3.7-alpine'" echo " DOCKER_TARGET A specific target to build." echo " It's currently not possible to pass multiple targets." echo " Default: main ldap" @@ -153,6 +153,13 @@ if [ ! -f "${DOCKERFILE}" ]; then fi fi +### +# Determining the value for DOCKER_FROM +### +if [ -z "$DOCKER_FROM" ]; then + DOCKER_FROM="python:3.7-alpine" +fi + ### # Variables for labelling the docker image ### @@ -167,9 +174,9 @@ PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:] # Get the Git information from the netbox directory if [ -d "${NETBOX_PATH}/.git" ]; then - NETBOX_GIT_REF=$(cd ${NETBOX_PATH}; git rev-parse HEAD) - NETBOX_GIT_BRANCH=$(cd ${NETBOX_PATH}; git rev-parse --abbrev-ref HEAD) - NETBOX_GIT_URL=$(cd ${NETBOX_PATH}; git remote get-url origin) + NETBOX_GIT_REF=$(cd "${NETBOX_PATH}"; git rev-parse HEAD) + NETBOX_GIT_BRANCH=$(cd "${NETBOX_PATH}"; git rev-parse --abbrev-ref HEAD) + NETBOX_GIT_URL=$(cd "${NETBOX_PATH}"; git remote get-url origin) fi ### @@ -217,15 +224,18 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # composing the additional DOCKER_SHORT_TAG, # i.e. "v2.6.1" becomes "v2.6", # which is only relevant for version tags + # Also let "latest" follow the highest version ### if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then MAJOR=${BASH_REMATCH[1]} MINOR=${BASH_REMATCH[2]} TARGET_DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" + TARGET_DOCKER_LATEST_TAG="${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:latest" if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_SHORT_TAG="${TARGET_DOCKER_SHORT_TAG}-${DOCKER_TARGET}" + TARGET_DOCKER_LATEST_TAG="${TARGET_DOCKER_LATEST_TAG}-${DOCKER_TARGET}" fi fi @@ -233,6 +243,48 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # Proceeding to buils stage, except if `--push-only` is passed ### if [ "${2}" != "--push-only" ] ; then + ### + # Checking if the build is necessary, + # meaning build only if one of those values changed: + # - Python base image digest (Label: PYTHON_BASE_DIGEST) + # - netbox git ref (Label: NETBOX_GIT_REF) + # - netbox-docker git ref (Label: org.label-schema.vcs-ref) + ### + # Load information from registry (only for docker.io) + SHOULD_BUILD="false" + BUILD_REASON="" + if [ -z "${GH_ACTION}" ]; then + # Asuming non Github builds should always proceed + SHOULD_BUILD="true" + BUILD_REASON="${BUILD_REASON} interactive" + elif [ "$DOCKER_REGISTRY" = "docker.io" ]; then + source ./build-functions/get-public-image-config.sh + IFS=':' read -ra DOCKER_FROM_SPLIT <<< "${DOCKER_FROM}" + if ! [[ ${DOCKER_FROM_SPLIT[0]} =~ .*/.* ]]; then + # Need to use "library/..." for images the have no two part name + DOCKER_FROM_SPLIT[0]="library/${DOCKER_FROM_SPLIT[0]}" + fi + PYTHON_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}") + mapfile -t IMAGES_LAYERS_OLD < <(get_image_layers "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") + NETBOX_GIT_REF_OLD=$(get_image_label NETBOX_GIT_REF "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") + GIT_REF_OLD=$(get_image_label org.label-schema.vcs-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") + + if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then + SHOULD_BUILD="true" + BUILD_REASON="${BUILD_REASON} python" + fi + if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then + SHOULD_BUILD="true" + BUILD_REASON="${BUILD_REASON} netbox" + fi + if [ "${GIT_REF}" != "${GIT_REF_OLD}" ]; then + SHOULD_BUILD="true" + BUILD_REASON="${BUILD_REASON} netbox-docker" + fi + else + SHOULD_BUILD="true" + BUILD_REASON="${BUILD_REASON} no-check" + fi ### # Composing all arguments for `docker build` ### @@ -244,6 +296,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ) if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_SHORT_TAG}" ) + DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_LATEST_TAG}" ) fi # --label @@ -269,6 +322,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}" ) fi + if [ -n "${BUILD_REASON}" ]; then + BUILD_REASON=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< "$BUILD_REASON") + DOCKER_BUILD_ARGS+=( --label "BUILD_REASON=${BUILD_REASON}" ) + fi # --build-arg DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) @@ -287,25 +344,29 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### # Building the docker image ### - echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." - $DRY docker build "${DOCKER_BUILD_ARGS[@]}" . - echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" + if [ "${SHOULD_BUILD}" == "true" ]; then + echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." + echo " Build reason set to: ${BUILD_REASON}" + $DRY docker build "${DOCKER_BUILD_ARGS[@]}" . + echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" + echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG}'" + $DRY docker inspect "${TARGET_DOCKER_TAG}" --format "{{json .Config.Labels}}" + else + echo "Build skipped because sources didn't change" + echo "::set-output name=skipped::true" + fi fi ### # Pushing the docker images if either `--push` or `--push-only` are passed ### if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then - echo "⏫ Inspecting labels on '${TARGET_DOCKER_TAG}'" - $DRY docker inspect "${TARGET_DOCKER_TAG}" --format "{{json .Config.Labels}}" - echo "⏫ Pushing '${TARGET_DOCKER_TAG}'" - $DRY docker push "${TARGET_DOCKER_TAG}" - echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." + source ./build-functions/docker-functions.sh + push_image_to_registry "${TARGET_DOCKER_TAG}" if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then - echo "⏫ Pushing '${TARGET_DOCKER_SHORT_TAG}'" - $DRY docker push "${TARGET_DOCKER_SHORT_TAG}" - echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_SHORT_TAG}'." + push_image_to_registry "${TARGET_DOCKER_SHORT_TAG}" + push_image_to_registry "${TARGET_DOCKER_LATEST_TAG}" fi fi done