.gitlab-ci.yml

variables:
  LANG: C.UTF-8
  IMAGE_NAME: linotp-build-test-environment_stretch
  BUILD_CODE_IMAGE: $REGISTRY_URL/$IMAGE_NAME:$CI_COMMIT_REF_NAME
  LATEST_CODE_IMAGE: $REGISTRY_URL/$IMAGE_NAME:$CI_PIPELINE_ID
  DOCKERFY_URL: http://avocado.corp.lsexperts.de/jenkins/dockerfy/dockerfy
  DEBIAN_MIRROR: ftp.de.debian.org
  DEPENDENCY_SOURCE: http://avocado.corp.lsexperts.de/deb-repo
  DEPENDENCY_SOURCE_COMPONENT: kideps
  DEPENDENCY_GPG_KEYID: 47868600E7533829
  DEPENDENCY_GPG_KEYID_URL: ""
  DOCKER_TAGS: $CI_PIPELINE_ID
  CI_IMAGE_PREFIX: $REGISTRY_URL/$CI_PROJECT_NAME/ci
  NOSE_COVER_PACKAGE: linotp
  DOCKER_COMPOSE_VERSION: "1.22.0"

stages:
  - build-test-environment
  - test
  - coverage-report
  - build-debian-packages
  - build-linotp-docker-image
  - selenium-test
  - upload-debian-packages

.docker-registry: &docker-registry
  before_script:
    - docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD $REGISTRY_URL

.build-test-environment-debian:
  stage: build-test-environment
  image: docker:latest
  <<: *docker-registry
  script:
    # pull the old image to speedup the build process, the hope is
    # that the modified time will always be the same unless the file
    # has actually been changed
    - DOCKERFILE_CACHE_KEY=$(sha256sum ci/Dockerfile.debian-base | cut -d ' ' -f1)
    - docker pull "$CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${DOCKERFILE_CACHE_KEY}" || true
    # update the base image, if there was no change this is a noop. If there was a change we will start
    # rebuilding the image. Worst case we downloaded an older image
    # that was based on an outdated image and rebuild it from
    # scratch. This should happen at most once a day.
    - docker pull debian:${DEBIAN_RELEASE_NAME} || true
    - mkdir -p /var/lib/empty
    - >
      docker build
      --cache-from="$CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${DOCKERFILE_CACHE_KEY}"
      --build-arg=BASE_IMAGE="debian:${DEBIAN_RELEASE_NAME}"
      --build-arg="DEBIAN_RELEASE_NAME=${DEBIAN_RELEASE_NAME}"
      --build-arg="DEPENDENCY_SOURCE=${DEPENDENCY_SOURCE}"
      --build-arg="DEPENDENCY_COMPONENT=${DEPENDENCY_COMPONENT}"
      --build-arg="DEPENDENCY_GPG_KEYID=${DEPENDENCY_GPG_KEYID}"
      --build-arg="DEPENDENCY_GPG_KEY_URL=${DEPENDENCY_GPG_KEY_URL}"
      --tag="$CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}"
      --tag="$CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${DOCKERFILE_CACHE_KEY}"
      -f ci/Dockerfile.debian-base /var/lib/empty
    - docker push "$CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}"
    - docker push "$CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${DOCKERFILE_CACHE_KEY}"
    - docker images

build-test-environment:debian-stretch:
  extends: .build-test-environment-debian
  variables:
    DEPENDENCY_SOURCE: http://avocado.corp.lsexperts.de/deb-repo
    DEPENDENCY_COMPONENT: kideps
    DEPENDENCY_GPG_KEYID: ""
    DEPENDENCY_GPG_KEY_URL: http://avocado.corp.lsexperts.de/deb-repo/archive-key.asc
    DEBIAN_RELEASE_NAME: stretch

build-test-environment:debian-jessie:
  extends: .build-test-environment-debian
  variables:
    DEPENDENCY_SOURCE: http://www.linotp.org/apt/debian
    DEPENDENCY_COMPONENT: linotp
    DEPENDENCY_GPG_KEYID: 913DFF12F86258E5
    DEBIAN_RELEASE_NAME: jessie

build-test-environment:debian-buster:
  extends: .build-test-environment-debian
  allow_failure: true
  variables:
    DEPENDENCY_SOURCE: ""
    DEPENDENCY_GPG_KEYID: ""
    DEBIAN_RELEASE_NAME: buster

.test-debian:
  image: $CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}
  before_script:
    - cd linotpd/src
    - python --version
    - pip install --upgrade setuptools
    - pip install wheel
    - pip install freezegun
    - pip install mock
    - pip install -U cffi
    - pip install coverage
    - pip install bcrypt
    - pip freeze
    - rm .coverage || true # remove coverage file in case it exists (e.g. re-runs)
    #- pip install -e .[tests]
    - python setup.py develop
    - python tools/linotp-create-enckey -f test.ini
    - python tools/linotp-create-auditkeys -f test.ini
    - export NOSE_WITH_COVERAGE=1
    - export NOSE_COVER_INCLUSIVE=1
    - export NOSE_COVER_DIR=${CI_PROJECT_DIR}/coverage/${DEBIAN_RELEASE_NAME}/
    - export COVERAGE_FILE=${NOSE_COVER_DIR}/unit
    - mkdir -p ${NOSE_COVER_DIR}
  artifacts:
    paths:
      - coverage/${DEBIAN_RELEASE_NAME}

unit-test:debian-jessie:
  extends: .test-debian
  stage: test
  variables:
    DEBIAN_RELEASE_NAME: jessie
  script:
    - nosetests -v linotp/tests/unit/

unit-test:debian-stretch:
  extends: .test-debian
  stage: test
  variables:
    DEBIAN_RELEASE_NAME: stretch
  script:
    - nosetests --with-pylons=test.ini -v linotp/tests/unit/

unit-test:debian-buster:
  extends: .test-debian
  allow_failure: true
  stage: test
  variables:
    DEBIAN_RELEASE_NAME: buster
  script:
    - nosetests --with-pylons=test.ini -v linotp/tests/unit/

.functional-test:
  stage: test
  image: $CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}
  services:
    - name: mysql:5.5
      alias: db
  variables:
    MYSQL_DATABASE: linotp_db
    MYSQL_ROOT_PASSWORD: linotppass
    MYSQL_USER: linotpuser
    MYSQL_PASSWORD: linotppass
    MYSQL_HOST: mysql
    SHELL: /bin/bash
  before_script:
    - test "${DEBIAN_RELEASE_NAME}" != "buster" && apt-get install --yes --no-install-recommends parallel mysql-client python-mysqldb python-imaging || apt-get install --yes --no-install-recommends python-pil default-mysql-client python-mysqldb
    - test "${DEBIAN_RELEASE_NAME}" == "jessie" && apt-get install --yes --no-install-recommends libmysqlclient-dev || apt-get install  --yes --no-install-recommends default-libmysqlclient-dev
    - cd linotpd/src
    - pip install --upgrade setuptools
    - pip install wheel
    - pip install freezegun
    - pip install mock
    - pip install mysql
    - pip install unittest2
    - pip install nose nose-testconfig
    - pip install pyrad
    - pip install -U cffi
    - pip install coverage
    - pip install bcrypt
    - pip freeze
    - python setup.py develop
    - python tools/linotp-create-enckey -f test.ini
    - python tools/linotp-create-auditkeys -f test.ini
    - cp encKey public.pem private.pem /tmp/
    - mkdir /tmp/config
    - cp config/dictionary /tmp/config
    - rm test.ini
    - export NOSE_WITH_COVERAGE=1
    - test "$CI_PIPELINE_SOURCE" != "schedule" && export NOSE_ATTRS='!nightly'
    - export NOSE_COVER_DIR=${CI_PROJECT_DIR}/coverage/${DEBIAN_RELEASE_NAME}/
    - mkdir -p ${NOSE_COVER_DIR}
    - find linotp/tests/functional -type f -iname 'test_*.py' >func-tests.txt
    - num_tests=`cat func-tests.txt | wc -l`
    - echo $num_tests
    # ocra2 test irrelevant and a damn long execution duration
    # So remove for commit triggered pipeline - only executed during nightly scheduled pipeline
    - test "$CI_PIPELINE_SOURCE" != "schedule" && sed -i '/test_ocra2/d' func-tests.txt
    - split --numeric-suffixes=1 -n r/5 func-tests.txt func_chunk_
    - cat func_chunk_01
    - cat func_chunk_02
    - cat func_chunk_03
    - cat func_chunk_04
    - cat func_chunk_05
  script:
    - bash -c "find linotp/tests/functional -type f -iname 'test_*.py' | parallel -kN2 -j$(nproc) ../../ci/run_functional.sh {#} {}"
  artifacts:
    paths:
      - coverage/${DEBIAN_RELEASE_NAME}/

.functional-special-test:
  extends: .functional-test
  stage: test
  variables:
    COVERAGE_PREFIX: func-special
  script:
    - bash -c "find linotp/tests/functional_special -type f -iname 'test_*.py' | parallel -kN2 -j$(nproc) ../../ci/run_functional.sh {#} {}"

func:jessie:
  extends: .functional-test
  stage: test
  parallel: 5
  script:
    - echo $CI_NODE_INDEX
    - bash -c "cat func_chunk_0$CI_NODE_INDEX | parallel -kN2 -j$(nproc) ../../ci/run_functional.sh {#} {}"
  variables:
    DEBIAN_RELEASE_NAME: jessie

func:stretch:
  extends: .functional-test
  stage: test
  parallel: 5
  script:
    - echo $CI_NODE_INDEX
    - bash -c "cat func_chunk_0$CI_NODE_INDEX | parallel -kN2 -j$(nproc) ../../ci/run_functional.sh {#} {}"
  variables:
    DEBIAN_RELEASE_NAME: stretch
  
functional-test:buster:
  extends: .functional-test
  allow_failure: true
  variables:
    DEBIAN_RELEASE_NAME: buster

functional-special-test:jessie:
  extends: .functional-special-test
  variables:
    DEBIAN_RELEASE_NAME: jessie

functional-special-test:stretch:
  extends: .functional-special-test
  variables:
    DEBIAN_RELEASE_NAME: stretch

functional-special-test:buster:
  extends: .functional-special-test
  allow_failure: true
  variables:
    DEBIAN_RELEASE_NAME: buster

.coverage-report:
  stage: coverage-report
  image: $CI_IMAGE_PREFIX/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}
  script:
    - pip install coverage
    - cd coverage/${DEBIAN_RELEASE_NAME}
    - rm .coverage || true # remove coverage file in case it exists (e.g. re-runs)
    - coverage combine *
    - coverage report | tee coverage.txt
    - coverage html
  artifacts:
    paths:
      - coverage/${DEBIAN_RELEASE_NAME}

coverage-report:jessie:
  extends: .coverage-report
  variables:
    DEBIAN_RELEASE_NAME: jessie

coverage-report:stretch:
  extends: .coverage-report
  variables:
    DEBIAN_RELEASE_NAME: stretch

coverage-report:buster:
  extends: .coverage-report
  allow_failure: true
  variables:
    DEBIAN_RELEASE_NAME: buster

.build-debs:
  stage: build-debian-packages
  image: debian:${DEBIAN_RELEASE_NAME}
  script:
    - apt-get update
    - apt-get install --yes --no-install-recommends equivs git-buildpackage lsb-release
    # Generate snapshot entries for debian changelogs
    - ci/update-debian-changelog adminclient/LinOTPAdminClientCLI/src
    - ci/update-debian-changelog linotpd/src
    - (cd linotpd/src && mk-build-deps --install --remove --tool "apt-get --yes --no-install-recommends")
    - make deb-install DESTDIR=pkg/apt/${DEBIAN_RELEASE_NAME}
  artifacts:
    paths:
      - pkg/apt/${DEBIAN_RELEASE_NAME}
    expire_in: 2 weeks


build:jessie:
  extends: .build-debs
  variables:
    DEBIAN_RELEASE_NAME: jessie

build:stretch:
  extends: .build-debs
  variables:
    DEBIAN_RELEASE_NAME: stretch

build:buster:
  extends: .build-debs
  allow_failure: true
  variables:
    DEBIAN_RELEASE_NAME: buster

.build-linotp-docker-image:
  stage: build-linotp-docker-image
  image: docker:latest
  retry:
    max: 2
  before_script:
    - apk add --no-cache make
    - docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD $REGISTRY_URL
  script:
    - export BUILDDIR=build
    - mkdir -p ${BUILDDIR}
    - make ${BUILDDIR}/dockerfy BUILDDIR=${BUILDDIR}
    - cp -r pkg/apt/${DEBIAN_RELEASE_NAME} ${BUILDDIR}/apt
    - cp ci/Dockerfile.linotp linotpd/src/linotp/tests/integration/testdata/se_mypasswd linotpd/src/config/*.tmpl linotpd/src/tools/linotp-create-htdigest $BUILDDIR
    - >
      docker build
      --build-arg=DEBIAN_MIRROR=$DEBIAN_MIRROR
      --build-arg=LINOTP_MIRROR=$LINOTP_MIRROR
      --build-arg=BASE_IMAGE="debian:${DEBIAN_RELEASE_NAME}"
      --build-arg="DEBIAN_RELEASE_NAME=${DEBIAN_RELEASE_NAME}"
      --build-arg="DEPENDENCY_SOURCE=${DEPENDENCY_SOURCE}"
      --build-arg="DEPENDENCY_COMPONENT=${DEPENDENCY_COMPONENT}"
      --build-arg="DEPENDENCY_GPG_KEYID=${DEPENDENCY_GPG_KEYID}"
      --build-arg="DEPENDENCY_GPG_KEY_URL=${DEPENDENCY_GPG_KEY_URL}"
      --tag="${CI_IMAGE_PREFIX}/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}" -f $BUILDDIR/Dockerfile.linotp $BUILDDIR
    - docker push "${CI_IMAGE_PREFIX}/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}"


build-linotp-docker-image:jessie:
  extends: .build-linotp-docker-image
  variables:
    DEPENDENCY_SOURCE: http://www.linotp.org/apt/debian
    DEPENDENCY_COMPONENT: linotp
    DEPENDENCY_GPG_KEYID: 913DFF12F86258E5
    DEBIAN_RELEASE_NAME: jessie

build-linotp-docker-image:stretch:
  extends: .build-linotp-docker-image
  variables:
    DEPENDENCY_SOURCE: http://avocado.corp.lsexperts.de/deb-repo
    DEPENDENCY_COMPONENT: kideps
    DEPENDENCY_GPG_KEYID: ""
    DEPENDENCY_GPG_KEY_URL: http://avocado.corp.lsexperts.de/deb-repo/archive-key.asc
    DEBIAN_RELEASE_NAME: stretch

build-linotp-docker-image:buster:
  extends: .build-linotp-docker-image
  allow_failure: true
  variables:
    DEPENDENCY_SOURCE: ""
    DEPENDENCY_GPG_KEYID: ""
    DEBIAN_RELEASE_NAME: buster

.e2e-test:
  stage: selenium-test
  image: docker:latest
  variables:
    INTEGRATION_TEST_CHROME: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/tests/integration/chrome:${CI_PIPELINE_ID}"
    DELETE_OLD_SCRIPT_EXECUTORS: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/tests/integration/script-executor"
    INTEGRATION_TEST_SCRIPT_EXECUTOR: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/tests/integration/script-executor:${CI_PIPELINE_ID}"
    DELETE_OLD_INTEGRATION_LINOTPS: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/tests/integration/linotp"
    INTEGRATION_TEST_LINOTP: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/tests/integration/linotp:${CI_PIPELINE_ID}"
    DELETE_OLD_LINOTPS: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/linotp"
    LINOTP_IMAGE: "${REGISTRY_URL}/${CI_PROJECT_NAME}/ci/${DEBIAN_RELEASE_NAME}/linotp:${CI_PIPELINE_ID}"
  before_script:
    - apk add --no-cache py-pip
    - pip install "docker-compose==${DOCKER_COMPOSE_VERSION}"
    - docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD $REGISTRY_URL
    - cp -r pkg/apt/${DEBIAN_RELEASE_NAME} linotpd/src/linotp/tests/integration/apt
    - cd linotpd/src/linotp/tests/integration
    - docker-compose -f docker-compose-gitlab.yml down --rmi all --remove-orphans
    - echo $INTEGRATION_TEST_LINOTP
    - echo $INTEGRATION_TEST_SCRIPT_EXECUTOR
    - echo $INTEGRATION_TEST_CHROME
    - echo $LINOTP_IMAGE
    - echo $DELETE_OLD_SCRIPT_EXECUTORS
    - echo $DELETE_OLD_INTEGRATION_LINOTPS
    - echo $DELETE_OLD_LINOTPS
    - docker images -a |  grep "$DELETE_OLD_LINOTPS" | awk '{print $3}' | xargs docker rmi -f || true
    - docker images -a |  grep "$DELETE_OLD_INTEGRATION_LINOTPS" | awk '{print $3}' | xargs docker rmi -f || true
    - docker images -a |  grep "$DELETE_OLD_SCRIPT_EXECUTORS" | awk '{print $3}' | xargs docker rmi -f || true
    - docker-compose -f docker-compose-gitlab.yml config
    - docker-compose -f docker-compose-gitlab.yml build --force-rm --no-cache
    - docker-compose -f docker-compose-gitlab.yml push --ignore-push-failures test-script-executor
    - docker-compose -f docker-compose-gitlab.yml push --ignore-push-failures test-chrome
    - docker-compose -f docker-compose-gitlab.yml push --ignore-push-failures test-linotp
  script:
    - docker-compose -f docker-compose-gitlab.yml run --rm test-script-executor
  after_script:
    - cd linotpd/src/linotp/tests/integration
    - docker-compose -f docker-compose-gitlab.yml down --rmi all --remove-orphans

e2e-1:jessie:mysql-5.5:
  retry:
    max: 1
  extends: .e2e-test
  script:
    - docker-compose -f docker-compose-gitlab.yml run -e TEST_BUNCH=integrationtests-bunch1 --rm test-script-executor
  variables:
    DEBIAN_RELEASE_NAME: jessie
    DB: mysql:5.5

e2e-2:jessie:mysql-5.5:
  retry:
    max: 1
  extends: .e2e-test
  script:
    - docker-compose -f docker-compose-gitlab.yml run -e TEST_BUNCH=integrationtests-bunch2 --rm test-script-executor
  variables:
    DEBIAN_RELEASE_NAME: jessie
    DB: mysql:5.5

e2e-1:jessie:mariadb:10.1:
  allow_failure: true
  extends: .e2e-test
  script:
    - docker-compose -f docker-compose-gitlab.yml run -e TEST_BUNCH=integrationtests-bunch1 --rm test-script-executor
  variables:
    DEBIAN_RELEASE_NAME: jessie
    DB: mariadb:10.1

e2e-2:jessie:mariadb-10.1:
  allow_failure: true
  extends: .e2e-test
  script:
    - docker-compose -f docker-compose-gitlab.yml run -e TEST_BUNCH=integrationtests-bunch2 --rm test-script-executor
  variables:
    DEBIAN_RELEASE_NAME: jessie
    DB: mariadb:10.1

e2e-1:stretch:mariadb-10.1:
  retry:
    max: 1
  extends: .e2e-test
  script:
    - docker-compose -f docker-compose-gitlab.yml run -e TEST_BUNCH=integrationtests-bunch1 --rm test-script-executor
  variables:
    DEBIAN_RELEASE_NAME: stretch
    DB: mariadb:10.1

e2e-2:stretch:mariadb-10.1:
  retry:
    max: 1
  extends: .e2e-test
  script:
    - docker-compose -f docker-compose-gitlab.yml run -e TEST_BUNCH=integrationtests-bunch2 --rm test-script-executor
  variables:
    DEBIAN_RELEASE_NAME: stretch
    DB: mariadb:10.1

# Base rule for uploading deb packages to development repository.
# We use scp to upload all the files to an incoming directory.
.upload-debs:
  stage: upload-debian-packages
  image: debian:latest
  before_script:
    # Ensure required variables have been set
    - test -n "${DEV_REPO_HOST}"
    - test -n "${DEV_REPO_KNOWN_HOSTS}"
    - test -n "${DEV_REPO_SSH_SUBMIT_KEY}"
    # Install dependencies
    - apt-get update && apt-get install -y --no-install-recommends devscripts openssh-client
    # Configure ssh
    - eval $(ssh-agent -s)
    - echo "$DEV_REPO_SSH_SUBMIT_KEY" | tr -d '\r' | ssh-add - >/dev/null
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - echo "CheckHostIP no" >>~/.ssh/config
    - echo "$DEV_REPO_KNOWN_HOSTS" >~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
  script:
    - find artifacts -ls
    # scp all files referenced by the changes files to the repository
    - dcmd scp artifacts/*.changes dists@$DEV_REPO_HOST:deb-repo/incoming

# Allow a manual step to upload packages directly from a pipeline
# for testing
upload-debs:manual:
  extends: .upload-debs
  when: manual
  except:
    - schedules

# Upload packages automatically when tagged
upload-debs:tags:
  extends: .upload-debs
  only:
    - tags
  except:
    - schedules

# Upload packages automatically when merged to stable branch
upload-debs:stable-branch:
  extends: .upload-debs
  only:
    variables:
      - $CI_BRANCH =~ /^branch\//
  except:
    - tags
    - schedules