From d779676724dcb78d5b9149896aa8116e81cb6fdf Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 25 Oct 2016 18:43:00 -0400 Subject: [PATCH 1/8] Set the locale to avoid unicode tests from failing. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82d4647ed7..c91474266c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ services: - mysql:latest - postgres before_script: + - export LANG=en_US.UTF-8 - apt-get update -qq - apt-get install -qq python-dev python-pip gcc gnupg1 tesseract-ocr tesseract-ocr-deu ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice variables: From 7ead59a408cdff2ee00b50a810265e0110509301 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 25 Oct 2016 19:05:24 -0400 Subject: [PATCH 2/8] Force locale previous of tests (GitLab CI). --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c91474266c..eed0b5ed55 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ services: - mysql:latest - postgres before_script: - - export LANG=en_US.UTF-8 - apt-get update -qq - apt-get install -qq python-dev python-pip gcc gnupg1 tesseract-ocr tesseract-ocr-deu ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice variables: @@ -18,7 +17,7 @@ test:mysql: - pip install mysql-python - apt-get install -qq mysql-client - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "ALTER DATABASE $MYSQL_DATABASE CHARACTER SET utf8 COLLATE utf8_unicode_ci;" - - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations + - LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN tags: - mysql @@ -27,12 +26,12 @@ test:postgres: - apt-get install -qq libpq-dev - pip install -r requirements/testing.txt - pip install psycopg2 - - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations + - LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN tags: - postgres test:sqlite: script: - pip install -r requirements/testing.txt - - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci --nomigrations + - LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN From 9d527ae423b835e31c6cfd556be16300a65b0765 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Oct 2016 01:41:22 -0400 Subject: [PATCH 3/8] Improve the UTF locale generation for the GitLab CI image. --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eed0b5ed55..dca09a147b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,10 @@ services: - mysql:latest - postgres before_script: + - echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale + - locale-gen en_US.UTF-8 + - update-locale LANG=en_US.UTF-8 + - export LC_ALL=en_US.UTF-8 - apt-get update -qq - apt-get install -qq python-dev python-pip gcc gnupg1 tesseract-ocr tesseract-ocr-deu ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice variables: @@ -17,7 +21,7 @@ test:mysql: - pip install mysql-python - apt-get install -qq mysql-client - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "ALTER DATABASE $MYSQL_DATABASE CHARACTER SET utf8 COLLATE utf8_unicode_ci;" - - LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations + - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN tags: - mysql @@ -26,12 +30,12 @@ test:postgres: - apt-get install -qq libpq-dev - pip install -r requirements/testing.txt - pip install psycopg2 - - LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations + - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN tags: - postgres test:sqlite: script: - pip install -r requirements/testing.txt - - LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci --nomigrations + - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN From 0b5edb4ad61a0492897376b60c69849cf9ec76c9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Oct 2016 02:16:44 -0400 Subject: [PATCH 4/8] Add support to ignore certain temporary file patterns to the TempfileCheckMixin --- mayan/apps/common/tests/mixins.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py index cd8be439a0..d43db85616 100644 --- a/mayan/apps/common/tests/mixins.py +++ b/mayan/apps/common/tests/mixins.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import glob import os import psutil @@ -32,8 +33,29 @@ class ContentTypeCheckMixin(object): class TempfileCheckMixin(object): + # Ignore the jvmstat instrumentation file + ignore_globs = ('hsperfdata_*',) + def _get_temporary_entries_count(self): - return len(os.listdir(setting_temporary_directory.value)) + ignored_result = [] + + # Expand globs by joining the temporary directory and then flattening + # the list of lists into a single list + for item in self.ignore_globs: + ignored_result.extend( + glob.glob( + os.path.join(setting_temporary_directory.value, item) + ) + ) + + # Remove the path and leave only the expanded filename + ignored_result = map(lambda x: os.path.split(x)[-1], ignored_result) + + return len( + set( + os.listdir(setting_temporary_directory.value) + ) - set(ignored_result) + ) def setUp(self): super(TempfileCheckMixin, self).setUp() From 72980f09a60f187de5cb8245d48d02c21de48f5e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Oct 2016 04:05:32 -0400 Subject: [PATCH 5/8] Update TempfileCheckMixin to show list of oprhaned files. --- mayan/apps/common/tests/mixins.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py index d43db85616..f4cc360953 100644 --- a/mayan/apps/common/tests/mixins.py +++ b/mayan/apps/common/tests/mixins.py @@ -36,7 +36,7 @@ class TempfileCheckMixin(object): # Ignore the jvmstat instrumentation file ignore_globs = ('hsperfdata_*',) - def _get_temporary_entries_count(self): + def _get_temporary_entries(self): ignored_result = [] # Expand globs by joining the temporary directory and then flattening @@ -51,22 +51,23 @@ class TempfileCheckMixin(object): # Remove the path and leave only the expanded filename ignored_result = map(lambda x: os.path.split(x)[-1], ignored_result) - return len( - set( - os.listdir(setting_temporary_directory.value) - ) - set(ignored_result) - ) + return set( + os.listdir(setting_temporary_directory.value) + ) - set(ignored_result) def setUp(self): super(TempfileCheckMixin, self).setUp() - self._temporary_items = self._get_temporary_entries_count() + self._temporary_items = self._get_temporary_entries() def tearDown(self): + final_temporary_items = self._get_temporary_entries() self.assertEqual( - self._temporary_items, self._get_temporary_entries_count(), - msg='Orphan temporary file. The number of temporary file and ' + self._temporary_items, final_temporary_items, + msg='Orphan temporary file. The number of temporary files and/or ' 'directories at the start and at the end of the test are not the ' - 'same.' + 'same. Orphan entries: {}'.format( + ','.join(final_temporary_items-self._temporary_items) + ) ) super(TempfileCheckMixin, self).tearDown() From fa7c6ae8d7782b734aab5fd97916d949437dca20 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Oct 2016 04:06:00 -0400 Subject: [PATCH 6/8] Set the GitLab ci mysql service db server character set to utf8mb4. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dca09a147b..53d81eba49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ test:mysql: - pip install -r requirements/testing.txt - pip install mysql-python - apt-get install -qq mysql-client - - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "ALTER DATABASE $MYSQL_DATABASE CHARACTER SET utf8 COLLATE utf8_unicode_ci;" + - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;" - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN tags: From 7b9a2bcfccc303ab4bcfa3ab144617f98b6b68f3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Oct 2016 04:26:22 -0400 Subject: [PATCH 7/8] Ignore GitLab's CI .config temporary file. --- mayan/apps/common/tests/mixins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py index f4cc360953..79f0d06d60 100644 --- a/mayan/apps/common/tests/mixins.py +++ b/mayan/apps/common/tests/mixins.py @@ -33,8 +33,8 @@ class ContentTypeCheckMixin(object): class TempfileCheckMixin(object): - # Ignore the jvmstat instrumentation file - ignore_globs = ('hsperfdata_*',) + # Ignore the jvmstat instrumentation and GitLab's CI .config files + ignore_globs = ('hsperfdata_*', '.config') def _get_temporary_entries(self): ignored_result = [] From 7cc5064df30e8f2fa4cd5b38f96efcbf1d9c2000 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Oct 2016 04:43:11 -0400 Subject: [PATCH 8/8] Add curl to the GitLab CI script to be able to send the coverage report. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53d81eba49..c9c0de6789 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ before_script: - update-locale LANG=en_US.UTF-8 - export LC_ALL=en_US.UTF-8 - apt-get update -qq - - apt-get install -qq python-dev python-pip gcc gnupg1 tesseract-ocr tesseract-ocr-deu ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice + - apt-get install -qq curl python-dev python-pip gcc gnupg1 tesseract-ocr tesseract-ocr-deu ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice variables: POSTGRES_DB: "mayan_edms" POSTGRES_PASSWORD: "postgres"