.gitlab-ci.yml

stages:
  - stage_test
  - stage_build_python
  - stage_build_docker
  - stage_build_documentation
  - stage_push_python
  - stage_push_docker
  - stage_push_documentation
  - stage_deploy

# Docker

job_docker_build:
  image: docker:20-dind
  only:
    - releases/all_major
    - releases/all_minor
    - releases/docker_major
    - releases/docker_minor
    - tests/all
    - tests/docker
  script:
    - docker build --build-arg APT_PROXY=$APT_PROXY --build-arg PIP_INDEX_URL=$PIP_INDEX_URL --build-arg PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST --file docker/Dockerfile --pull --tag "mayanedms" .
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - docker tag "mayanedms" "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - SERVICE_POSTGRES_IP=$(cat /etc/hosts | awk '{if ($2 == "postgres") print $1;}')
    - docker run --rm --add-host="postgres:$SERVICE_POSTGRES_IP" -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql_psycopg2','NAME':'mayan','USER':'mayan','PASSWORD':'mayanuserpass','HOST':'postgres'}}" "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" run_tests
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - docker push "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker logout $CI_REGISTRY
  services:
    - docker:20-dind
    - postgres:10.15-alpine
  stage: stage_build_docker
  tags:
    - postgres
  variables:
    POSTGRES_DB: "mayan"
    POSTGRES_PASSWORD: "mayanuserpass"
    POSTGRES_USER: "mayan"

job_docker_push_major:
  before_script:
    - apk add --no-cache python3
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
  image: docker:20-dind
  only:
    - releases/all_major
    - releases/docker_major
  script:
    - PYTHON_BIN=/usr/bin/python3
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - docker pull "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
    - VERSION_SCRIPT_BIN=contrib/scripts/version.py
    - MAJOR_VERSION=$($PYTHON_BIN $VERSION_SCRIPT_BIN $DOCKER_VERSION major);
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:s"$MAJOR_VERSION"
    - MINOR_VERSION=$($PYTHON_BIN $VERSION_SCRIPT_BIN $DOCKER_VERSION minor);
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
    - docker logout $CI_REGISTRY
    - docker login --username "$DOCKER_HUB_USERNAME" --password "$DOCKER_HUB_PASSWORD" https://registry-1.docker.io
    - docker push registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker push registry-1.docker.io/mayanedms/mayanedms:latest
    - docker push registry-1.docker.io/mayanedms/mayanedms:s"$MAJOR_VERSION"
    - docker push registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
  services:
    - docker:20-dind
  stage: stage_push_docker

job_docker_push_minor:
  before_script:
    - apk add --no-cache python3
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
  image: docker:20-dind
  only:
    - releases/all_minor
    - releases/docker_minor
  script:
    - PYTHON_BIN=/usr/bin/python3
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - docker pull "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
    - VERSION_SCRIPT_BIN=contrib/scripts/version.py
    - MINOR_VERSION=$($PYTHON_BIN $VERSION_SCRIPT_BIN $DOCKER_VERSION minor);
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
    - docker logout $CI_REGISTRY
    - docker login --username "$DOCKER_HUB_USERNAME" --password "$DOCKER_HUB_PASSWORD" https://registry-1.docker.io
    - docker push registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker push registry-1.docker.io/mayanedms/mayanedms:latest
    - docker push registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
  services:
    - docker:20-dind
  stage: stage_push_docker

job_docker_nightly:
  before_script:
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
  image: docker:20-dind
  only:
    - releases/nightly
    - releases/staging
  script:
    - docker build --build-arg APT_PROXY=$APT_PROXY --build-arg PIP_INDEX_URL=$PIP_INDEX_URL --build-arg PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST --file docker/Dockerfile --pull --tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
    - docker logout $CI_REGISTRY
  services:
    - docker:20-dind
  stage: stage_build_docker

job_docker_testing:
  before_script:
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
  image: docker:20-dind
  only:
    - releases/testing
  script:
    - PYTHON_BIN=/usr/bin/python3
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - docker build --build-arg APT_PROXY=$APT_PROXY --build-arg PIP_INDEX_URL=$PIP_INDEX_URL --build-arg PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST --file docker/Dockerfile --pull --tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" .
    - docker push "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker logout $CI_REGISTRY
  services:
    - docker:20-dind
  stage: stage_build_docker

# Documentation

job_documentation_build:
  artifacts:
    expire_in: 2 hr
    paths:
    - docs
  before_script:
    - apt-get update
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get install --yes enchant make python3-venv python3-dev
    - python3 -m venv venv
    - . venv/bin/activate
  image: debian:10.10-slim
  only:
    - releases/all_major
    - releases/documentation
    - tests/all
    - tests/documentation
  script:
    - pip install --requirement requirements/documentation.txt
    - cd docs
    - make html
  stage: stage_build_documentation

job_documentation_push:
  before_script:
    - mkdir --parents ~/.ssh
    - chmod 700 ~/.ssh
    - echo "$DOCUMENTATION_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - 'which ssh-agent || ( apt-get update --yes && apt-get install --yes openssh-client rsync )'
    - eval $(ssh-agent -s)
    - echo "$DOCUMENTATION_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
  image: debian:10.10-slim
  only:
    - releases/all_major
    - releases/documentation
  script:
    - ssh mayandocs@docs.mayan-edms.com "rm -Rf /home/rosarior/webapps/mayan_docs/*"
    - rsync -avzP --progress -e ssh docs/_build/html/* mayandocs@docs.mayan-edms.com:/home/rosarior/webapps/mayan_docs
  stage: stage_push_documentation

# Python

job_python_build:
  artifacts:
    expire_in: 2 hr
    paths:
    - dist
  before_script:
    - set -a && . ./config.env && set +a
    - apt-get update
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get install --yes make python3-venv python3-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
  image: debian:10.10-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/python_major
    - releases/python_minor
    - releases/testing
  script:
    - pip install --requirement requirements/build.txt
    - make python-wheel
  stage: stage_build_python

job_python_push_pypi:
  before_script:
    - set -a && . ./config.env && set +a
    - apt-get update
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get install --yes make python3-venv python3-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
  image: debian:10.10-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/python_major
    - releases/python_minor
    - releases/testing
  script:
    - pip install --requirement requirements/build.txt
    - echo "$PYPIRC" > ~/.pypirc
    - twine upload dist/* -r pypi
  stage: stage_push_python

# Test upgrade from the latest public version to the local version

.job_base_upgrade_test: &job_base_upgrade_test
  before_script:
    - set -a && . ./config.env && set +a
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - echo "deb http://deb.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list
    - echo "deb-src http://deb.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list
    - apt-get update
    - apt-get install --yes curl exiftool gcc ghostscript gnupg1 libjpeg-dev libmagic1 libpng-dev libtiff-dev make poppler-utils python3-venv python3-dev tesseract-ocr
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - pip install "mayan-edms<$DOCKER_VERSION"
    - pip install --requirement requirements/build.txt --requirement requirements/testing-base.txt --requirement requirements/build.txt
    - make python-wheel
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apt/archives/
  image: debian:10.10-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/docker_major
    - releases/docker_minor
    - releases/python_major
    - releases/python_minor
    - releases/staging
    - tests/all
    - tests/python
  stage: stage_test

job_mysql_upgrade_test:
  <<: *job_base_upgrade_test
  script:
    - apt-get install --yes default-libmysqlclient-dev default-mysql-client libssl-dev
    - . venv/bin/activate
    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
    - mysql --host="$MYSQL_PORT_3306_TCP_ADDR" --port="$MYSQL_PORT_3306_TCP_PORT" --user=root --password="$MYSQL_ENV_MYSQL_ROOT_PASSWORD" --execute="set global character_set_server=utf8mb4;" --ssl=TRUE
    - export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.mysql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'$MYSQL_PORT_3306_TCP_ADDR','OPTIONS':{'ssl':True}}}"
    - mayan-edms.py initialsetup --settings=mayan.settings.testing.gitlab-ci
    - echo "from mayan.apps.documents.models import DocumentType;from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH;DocumentType.objects.first().new_document(file_object=open(file=TEST_SMALL_DOCUMENT_PATH, mode='rb'))" | mayan-edms.py shell
    - pip install dist/mayan_edms-*.whl
    - mayan-edms.py performupgrade --settings=mayan.settings.testing.gitlab-ci
  services:
    - mysql:8.0
  tags:
    - mysql
  variables:
    MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
    MYSQL_DATABASE: "mayan"
    MYSQL_PASSWORD: "mayanuserpass"
    MYSQL_USER: "mayan"

job_postgres_upgrade_test:
  <<: *job_base_upgrade_test
  script:
    - apt-get install --yes libpq-dev
    - . venv/bin/activate
    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
    - export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'postgres'}}"
    - mayan-edms.py initialsetup --settings=mayan.settings.testing.gitlab-ci
    - echo "from mayan.apps.documents.models import DocumentType;from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH;DocumentType.objects.first().new_document(file_object=open(file=TEST_SMALL_DOCUMENT_PATH, mode='rb'))" | mayan-edms.py shell
    - pip install dist/mayan_edms-*.whl
    - mayan-edms.py performupgrade --settings=mayan.settings.testing.gitlab-ci
  services:
    - postgres:10.15-alpine
  tags:
    - postgres
  variables:
    POSTGRES_DB: "mayan"
    POSTGRES_PASSWORD: "mayanuserpass"
    POSTGRES_USER: "mayan"

job_sqlite_upgrade_test:
  <<: *job_base_upgrade_test
  script:
    - . venv/bin/activate
    - mayan-edms.py initialsetup --settings=mayan.settings.testing.gitlab-ci
    - echo "from mayan.apps.documents.models import DocumentType;from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH;DocumentType.objects.first().new_document(file_object=open(file=TEST_SMALL_DOCUMENT_PATH, mode='rb'))" | mayan-edms.py shell
    - pip install dist/mayan_edms-*.whl
    - mayan-edms.py performupgrade --settings=mayan.settings.testing.gitlab-ci

# Run the test suit

.job_base_test: &job_base_test
  before_script:
    - set -a && . ./config.env && set +a
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - echo "deb http://deb.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list
    - echo "deb-src http://deb.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list
    - apt-get update
    - apt-get install --yes --no-install-recommends --target-release buster-backports libreoffice-calc-nogui libreoffice-draw-nogui libreoffice-impress-nogui libreoffice-math-nogui libreoffice-writer-nogui
    - apt-get install --yes --no-install-recommends curl exiftool gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev make poppler-utils python3-venv python3-dev sane-utils tesseract-ocr tesseract-ocr-deu
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - pip install --requirement requirements.txt --requirement requirements/testing-base.txt --requirement requirements/build.txt
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apt/archives/
  image: debian:10.10-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/docker_major
    - releases/docker_minor
    - releases/python_major
    - releases/python_minor
    - releases/staging
    - tests/all
    - tests/python
  stage: stage_test

job_postgres_test:
  <<: *job_base_test
  script:
    - apt-get install --yes libpq-dev
    - . venv/bin/activate
    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
    - export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'postgres'}}"
    - make test-all SETTINGS=mayan.settings.testing.gitlab-ci
    - make test-all-migrations SETTINGS=mayan.settings.testing.gitlab-ci
  services:
    - postgres:10.15-alpine
  tags:
    - postgres
  variables:
    POSTGRES_DB: "mayan"
    POSTGRES_PASSWORD: "mayanuserpass"
    POSTGRES_USER: "mayan"

job_sqlite_test:
  <<: *job_base_test
  script:
    - . venv/bin/activate
    - make test-all SETTINGS=mayan.settings.testing.gitlab-ci
    - make test-all-migrations SETTINGS=mayan.settings.testing.gitlab-ci

# Deploy

job_demo_deploy:
  before_script:
    - mkdir --parents ~/.ssh
    - echo "${DEMO_SSH_KNOWN_HOST}" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - echo "${DEMO_PRIVATE_KEY}" > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
  environment:
    name: demo
  script:
    - scp contrib/scripts/install/docker.sh root@$DEMO_IP_ADDRESS:~
    - ssh root@$DEMO_IP_ADDRESS "DELETE_VOLUMES=true ./docker.sh"
  stage: stage_deploy
  when: manual

job_staging_deploy:
  before_script:
    - mkdir --parents ~/.ssh
    - echo "${STAGING_SSH_KNOWN_HOST}" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - echo "${STAGING_PRIVATE_KEY}" > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
  environment:
    name: staging
  script:
    - scp contrib/scripts/install/docker.sh root@$STAGING_IP_ADDRESS:~
    - ssh root@$STAGING_IP_ADDRESS "DOCKER_MAYAN_IMAGE=registry.gitlab.com/mayan-edms/mayan-edms:staging ./docker.sh"
  stage: stage_deploy
  when: manual