From 75d20d1b57707fe43a8cfc45a7ef8743e089e10b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 31 Oct 2019 12:47:11 -0400 Subject: [PATCH 01/10] Backport Docker composer makefile targets Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docker/Makefile | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 4866d66974..adb5b1bb37 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ - Add makefile target to launch a production staging Docker image. - Improve duplicated document list view logic to not show documents with trashed duplicates. +- Backport Docker composer makefile targets. 3.2.8 (2019-10-01) ================== diff --git a/docker/Makefile b/docker/Makefile index 10ac96a8ec..d3e49fed4b 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -27,6 +27,15 @@ docker-runtest-cleanup: ## Delete the test container and the test volume. docker-runtest-all: ## Executed the test suite in a test container. docker run --rm run-tests +docker-compose-build: + docker-compose -f docker/docker-compose.yml -p mayan-edms build + +docker-compose-build-with-proxy: + docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) + +docker-compose-up: + docker-compose -f docker/docker-compose.yml -p mayan-edms up + docker-staging-network-create: @docker network rm mayan-staging || true docker network create mayan-staging From 1ca91e7ec5d59c6551b67e2d0783129cdc18a7ee Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 31 Oct 2019 12:57:40 -0400 Subject: [PATCH 02/10] Add cache invalidation test case mixins Add PermissionTestCaseMixin and SmartSettingTestCaseMixin to better organize cache invalidation of both apps for tests. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 ++ mayan/apps/common/tests/base.py | 14 +++++--------- mayan/apps/permissions/tests/mixins.py | 8 +++++++- mayan/apps/smart_settings/tests/mixins.py | 6 ++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index adb5b1bb37..340e5bacf6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,6 +31,8 @@ - Improve duplicated document list view logic to not show documents with trashed duplicates. - Backport Docker composer makefile targets. +- Add PermissionTestCaseMixin and SmartSettingTestCaseMixin to better + organize cache invalidation of both apps for tests. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/common/tests/base.py b/mayan/apps/common/tests/base.py index 3c352df9df..1a49c23867 100644 --- a/mayan/apps/common/tests/base.py +++ b/mayan/apps/common/tests/base.py @@ -5,8 +5,8 @@ from django.test import TestCase from django_downloadview import assert_download_response from mayan.apps.acls.tests.mixins import ACLTestCaseMixin -from mayan.apps.permissions.classes import Permission -from mayan.apps.smart_settings.classes import Namespace +from mayan.apps.permissions.tests.mixins import PermissionTestCaseMixin +from mayan.apps.smart_settings.tests.mixins import SmartSettingsTestCaseMixin from mayan.apps.user_management.tests.mixins import UserTestMixin from .mixins import ( @@ -21,19 +21,15 @@ from .mixins import ( class BaseTestCase( SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, RandomPrimaryKeyModelMonkeyPatchMixin, ACLTestCaseMixin, - ModelTestCaseMixin, OpenFileCheckTestCaseMixin, - TempfileCheckTestCasekMixin, UserTestMixin, TestCase + ModelTestCaseMixin, OpenFileCheckTestCaseMixin, PermissionTestCaseMixin, + SmartSettingsTestCaseMixin, TempfileCheckTestCasekMixin, UserTestMixin, + TestCase ): """ This is the most basic test case class any test in the project should use. """ assert_download_response = assert_download_response - def setUp(self): - super(BaseTestCase, self).setUp() - Namespace.invalidate_cache_all() - Permission.invalidate_cache() - class GenericViewTestCase( ClientMethodsTestCaseMixin, ContentTypeCheckTestCaseMixin, diff --git a/mayan/apps/permissions/tests/mixins.py b/mayan/apps/permissions/tests/mixins.py index 146746e44a..4270626c0c 100644 --- a/mayan/apps/permissions/tests/mixins.py +++ b/mayan/apps/permissions/tests/mixins.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from ..classes import PermissionNamespace +from ..classes import Permission, PermissionNamespace from ..models import Role from .literals import ( @@ -45,6 +45,12 @@ class PermissionTestMixin(object): ) +class PermissionTestCaseMixin(object): + def setUp(self): + super(PermissionTestCaseMixin, self).setUp() + Permission.invalidate_cache() + + class RoleAPIViewTestMixin(object): def _request_test_role_create_api_view(self, extra_data=None): data = { diff --git a/mayan/apps/smart_settings/tests/mixins.py b/mayan/apps/smart_settings/tests/mixins.py index 83cadcb4c3..2991121903 100644 --- a/mayan/apps/smart_settings/tests/mixins.py +++ b/mayan/apps/smart_settings/tests/mixins.py @@ -5,6 +5,12 @@ from ..classes import Namespace from .literals import TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME +class SmartSettingsTestCaseMixin(object): + def setUp(self): + super(SmartSettingsTestCaseMixin, self).setUp() + Namespace.invalidate_cache_all() + + class SmartSettingTestMixin(object): def _create_test_settings_namespace(self): try: From 9d883e455ec367b8cf759cddce4fc664db7fda24 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 31 Oct 2019 13:40:01 -0400 Subject: [PATCH 03/10] Add version attribute to namespaces Also dump the version of all the namespaces as children of SMART_SETTINGS_NAMESPACES. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 ++ mayan/apps/smart_settings/classes.py | 37 ++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 340e5bacf6..2877fb38fa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,8 @@ - Backport Docker composer makefile targets. - Add PermissionTestCaseMixin and SmartSettingTestCaseMixin to better organize cache invalidation of both apps for tests. +- Add a version attribute to setting namespace. These are dumped + as SMART_SETTINGS_NAMESPACES. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608..516d3e81d6 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -22,6 +22,7 @@ from django.utils.encoding import ( ) logger = logging.getLogger(__name__) +SMART_SETTINGS_NAMESPACES_NAME = 'SMART_SETTINGS_NAMESPACES' @python_2_unicode_compatible @@ -40,26 +41,35 @@ class Namespace(object): exception ) + @classmethod + def get(cls, name): + return cls._registry[name] + @classmethod def get_all(cls): return sorted(cls._registry.values(), key=lambda x: x.label) @classmethod - def get(cls, name): - return cls._registry[name] + def get_namespace_config(cls, name): + return cls.get_namespaces_config().get(name, {}) + + @classmethod + def get_namespaces_config(cls): + return getattr(settings, SMART_SETTINGS_NAMESPACES_NAME, {}) @classmethod def invalidate_cache_all(cls): for namespace in cls.get_all(): namespace.invalidate_cache() - def __init__(self, name, label): + def __init__(self, name, label, version='0001'): if name in self.__class__._registry: raise Exception( 'Namespace names must be unique; "%s" already exists.' % name ) self.name = name self.label = label + self.version = version self.__class__._registry[name] = self self._settings = [] @@ -69,6 +79,9 @@ class Namespace(object): def add_setting(self, **kwargs): return Setting(namespace=self, **kwargs) + def get_config_version(self): + return Namespace.get_namespace_config(name=self.name).get('version', None) + def invalidate_cache(self): for setting in self._settings: setting.invalidate_cache() @@ -123,7 +136,18 @@ class Setting(object): def dump_data(cls, filter_term=None, namespace=None): dictionary = {} + if not namespace: + namespace_dictionary = {} + for _namespace in Namespace.get_all(): + namespace_dictionary[_namespace.name] = { + 'version': _namespace.version + } + + dictionary[SMART_SETTINGS_NAMESPACES_NAME] = namespace_dictionary + for setting in cls.get_all(): + # If a namespace is specified, filter the list by that namespace + # otherwise return always True to include all (or not None == True) if (namespace and setting.namespace.name == namespace) or not namespace: if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term: dictionary[setting.global_name] = Setting.express_promises(setting.value) @@ -147,9 +171,12 @@ class Setting(object): ) @classmethod - def save_configuration(cls, path=settings.CONFIGURATION_FILEPATH): + def save_configuration(cls, path=None): + if not path: + path = settings.CONFIGURATION_FILEPATH + try: - with open(path, 'w') as file_object: + with open(path, mode='w') as file_object: file_object.write(cls.dump_data()) except IOError as exception: if exception.errno == errno.ENOENT: From f9fa3344d0d09026ea0b3cc118b7d30d325481cd Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 31 Oct 2019 13:42:04 -0400 Subject: [PATCH 04/10] Add savesettings command Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + .../management/commands/savesettings.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 mayan/apps/smart_settings/management/commands/savesettings.py diff --git a/HISTORY.rst b/HISTORY.rst index 2877fb38fa..2c868e820b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,7 @@ organize cache invalidation of both apps for tests. - Add a version attribute to setting namespace. These are dumped as SMART_SETTINGS_NAMESPACES. +- Add savesettings command. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/smart_settings/management/commands/savesettings.py b/mayan/apps/smart_settings/management/commands/savesettings.py new file mode 100644 index 0000000000..4d6e17fd19 --- /dev/null +++ b/mayan/apps/smart_settings/management/commands/savesettings.py @@ -0,0 +1,18 @@ +from __future__ import unicode_literals + +from django.core import management + +from ...classes import Setting + + +class Command(management.BaseCommand): + help = 'Save the current settings into the configuration file.' + + def add_arguments(self, parser): + parser.add_argument( + '--filepath', action='store', dest='filepath', + help='Filename and path where to save the configuration file.' + ) + + def handle(self, *args, **options): + Setting.save_configuration(path=options.get('filepath')) From a507bc89b7b6b905ef0b2bfa457eafa0739075bc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 1 Nov 2019 21:21:24 -0400 Subject: [PATCH 05/10] Add extra logging to the IMAP email source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitLab issue #682. Thanks to Patrick Hütter (@PatrickHuetter) for the report. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + mayan/apps/sources/models/email_sources.py | 47 +++++++++++++++++----- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2c868e820b..99d5c63666 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -36,6 +36,8 @@ - Add a version attribute to setting namespace. These are dumped as SMART_SETTINGS_NAMESPACES. - Add savesettings command. +- Add extra logging to the IMAP email source. GitLab issue #682. + Thanks to Patrick Hütter (@PatrickHuetter) for the report. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 4c07219088..d74347f206 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -20,6 +20,7 @@ from mayan.apps.documents.models import Document from mayan.apps.metadata.api import set_bulk_metadata from mayan.apps.metadata.models import MetadataType +from ..exceptions import SourceException from ..literals import ( DEFAULT_IMAP_MAILBOX, DEFAULT_METADATA_ATTACHMENT_NAME, DEFAULT_POP3_TIMEOUT, SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3, @@ -238,9 +239,22 @@ class IMAPEmail(EmailBaseModel): mailbox = imaplib.IMAP4(host=self.host, port=self.port) mailbox.login(user=self.username, password=self.password) - mailbox.select(mailbox=self.mailbox) + try: + mailbox.select(mailbox=self.mailbox) + except Exception as exception: + raise SourceException( + 'Error selecting mailbox: {}; {}'.format( + self.mailbox, exception + ) + ) + + try: + status, data = mailbox.search(None, 'NOT', 'DELETED') + except Exception as exception: + raise SourceException( + 'Error executing search command; {}'.format(exception) + ) - status, data = mailbox.search(None, 'NOT', 'DELETED') if data: messages_info = data[0].split() logger.debug('messages count: %s', len(messages_info)) @@ -250,15 +264,30 @@ class IMAPEmail(EmailBaseModel): status, data = mailbox.fetch( message_set=message_number, message_parts='(RFC822)' ) - EmailBaseModel.process_message( - source=self, message_text=data[0][1] - ) + try: + EmailBaseModel.process_message( + source=self, message_text=data[0][1] + ) + except Exception as exception: + raise SourceException( + 'Error processing message number: {}; {}'.format( + message_number, exception + ) + ) if not test: - mailbox.store( - message_set=message_number, command='+FLAGS', - flags=r'\Deleted' - ) + try: + mailbox.store( + message_set=message_number, command='+FLAGS', + flags=r'\Deleted' + ) + except Exception as exception: + raise SourceException( + 'Error executing IMAP store command ' + 'on message number {}; {}'.format( + message_number, exception + ) + ) mailbox.expunge() mailbox.close() From e14685e3e73767d2d68d22e331b304528928bb54 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 1 Nov 2019 21:23:51 -0400 Subject: [PATCH 06/10] Rename instances of the IMAP server variable Renamed from mailbox to server for clarity. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 ++ mayan/apps/sources/models/email_sources.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 99d5c63666..4c0c007e44 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -38,6 +38,8 @@ - Add savesettings command. - Add extra logging to the IMAP email source. GitLab issue #682. Thanks to Patrick Hütter (@PatrickHuetter) for the report. +- Rename all instances of the IMAP server from mailbox to + server for clarity. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index d74347f206..dd1b18b846 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -234,13 +234,13 @@ class IMAPEmail(EmailBaseModel): logger.debug('ssl: %s', self.ssl) if self.ssl: - mailbox = imaplib.IMAP4_SSL(host=self.host, port=self.port) + server = imaplib.IMAP4_SSL(host=self.host, port=self.port) else: - mailbox = imaplib.IMAP4(host=self.host, port=self.port) + server = imaplib.IMAP4(host=self.host, port=self.port) - mailbox.login(user=self.username, password=self.password) + server.login(user=self.username, password=self.password) try: - mailbox.select(mailbox=self.mailbox) + server.select(mailbox=self.mailbox) except Exception as exception: raise SourceException( 'Error selecting mailbox: {}; {}'.format( @@ -249,7 +249,7 @@ class IMAPEmail(EmailBaseModel): ) try: - status, data = mailbox.search(None, 'NOT', 'DELETED') + status, data = server.search(None, 'NOT', 'DELETED') except Exception as exception: raise SourceException( 'Error executing search command; {}'.format(exception) @@ -261,7 +261,7 @@ class IMAPEmail(EmailBaseModel): for message_number in messages_info: logger.debug('message_number: %s', message_number) - status, data = mailbox.fetch( + status, data = server.fetch( message_set=message_number, message_parts='(RFC822)' ) try: @@ -277,7 +277,7 @@ class IMAPEmail(EmailBaseModel): if not test: try: - mailbox.store( + server.store( message_set=message_number, command='+FLAGS', flags=r'\Deleted' ) @@ -289,9 +289,9 @@ class IMAPEmail(EmailBaseModel): ) ) - mailbox.expunge() - mailbox.close() - mailbox.logout() + server.expunge() + server.close() + server.logout() class POP3Email(EmailBaseModel): From f650a623b59c3dafbf3e712f7d963487c2a8b71c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 1 Nov 2019 21:39:44 -0400 Subject: [PATCH 07/10] Add book link in the about menu Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + mayan/apps/common/apps.py | 4 ++-- mayan/apps/common/icons.py | 1 + mayan/apps/common/links.py | 6 +++++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4c0c007e44..b75a6a3ef4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -40,6 +40,7 @@ Thanks to Patrick Hütter (@PatrickHuetter) for the report. - Rename all instances of the IMAP server from mailbox to server for clarity. +- Add book link in the about menu. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/common/apps.py b/mayan/apps/common/apps.py index 81494da8ff..fffbd109f3 100644 --- a/mayan/apps/common/apps.py +++ b/mayan/apps/common/apps.py @@ -22,7 +22,7 @@ from .handlers import ( handler_user_locale_profile_session_config, handler_user_locale_profile_create ) from .links import ( - link_about, link_current_user_locale_profile_edit, link_license, + link_about, link_book, link_current_user_locale_profile_edit, link_license, link_object_error_list_clear, link_setup, link_tools ) @@ -108,7 +108,7 @@ class CommonApp(MayanAppConfig): menu_about.bind_links( links=( - link_tools, link_setup, link_about, link_license, + link_tools, link_setup, link_about, link_book, link_license, ) ) diff --git a/mayan/apps/common/icons.py b/mayan/apps/common/icons.py index fc72d63d70..9fd1271e7b 100644 --- a/mayan/apps/common/icons.py +++ b/mayan/apps/common/icons.py @@ -11,6 +11,7 @@ icon_add_all = Icon( ) icon_assign_remove_add = Icon(driver_name='fontawesome', symbol='plus') icon_assign_remove_remove = Icon(driver_name='fontawesome', symbol='minus') +icon_book = Icon(driver_name='fontawesome', symbol='book') icon_confirm_form_submit = Icon(driver_name='fontawesome', symbol='check') icon_confirm_form_cancel = Icon(driver_name='fontawesome', symbol='times') icon_current_user_locale_profile_details = Icon( diff --git a/mayan/apps/common/links.py b/mayan/apps/common/links.py index 25a74bf903..1a769fd5b1 100644 --- a/mayan/apps/common/links.py +++ b/mayan/apps/common/links.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.navigation.classes import Link from .icons import ( - icon_about, icon_current_user_locale_profile_details, + icon_about, icon_book, icon_current_user_locale_profile_details, icon_current_user_locale_profile_edit, icon_documentation, icon_forum, icon_license, icon_object_error_list_with_icon, icon_setup, icon_source_code, icon_support, icon_tools @@ -35,6 +35,10 @@ def get_kwargs_factory(variable_name): link_about = Link( icon_class=icon_about, text=_('About this'), view='common:about_view' ) +link_book = Link( + icon_class=icon_book, tags='new_window', text=_('Get the book'), + url='https://mayan-edms.com/book/' +) link_current_user_locale_profile_details = Link( icon_class=icon_current_user_locale_profile_details, text=_('Locale profile'), From aa608c609477bdd0d1ad28edaa914da8adc26419 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 1 Nov 2019 21:41:06 -0400 Subject: [PATCH 08/10] Add exception handling to latest version check Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 ++ mayan/apps/dependencies/views.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b75a6a3ef4..f5f1e1292e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,6 +41,8 @@ - Rename all instances of the IMAP server from mailbox to server for clarity. - Add book link in the about menu. +- Add unknown exception handling when checking for the latest + version. 3.2.8 (2019-10-01) ================== diff --git a/mayan/apps/dependencies/views.py b/mayan/apps/dependencies/views.py index e5d9951e7e..7e970be5be 100644 --- a/mayan/apps/dependencies/views.py +++ b/mayan/apps/dependencies/views.py @@ -28,6 +28,12 @@ class CheckVersionView(SimpleView): 'It is not possible to determine the latest version ' 'available.' ) + except Exception as exception: + message = _( + 'Unexpected error trying to determine the latest version ' + 'available. Make sure your installation has a connection to ' + 'the internet; %s' + ) % exception else: message = _('Your version is up-to-date.') From 0ab9c187335a745869dd2ce516923dd043a35771 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 2 Nov 2019 02:11:35 -0400 Subject: [PATCH 09/10] Add docker install script release checklist step Signed-off-by: Roberto Rosario --- docs/chapters/development.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/chapters/development.rst b/docs/chapters/development.rst index 90e20552c9..ddca8b1b10 100644 --- a/docs/chapters/development.rst +++ b/docs/chapters/development.rst @@ -247,7 +247,7 @@ Mayan EDMS follows a simplified model layout based on Vincent Driessen's Working branches for unfinished and unmerged feature. Likely unstable, don't use in production. Once the feature is complete, it is merged into one of the versions branches and deleted. - + Special branches: ``releases/all`` @@ -317,7 +317,7 @@ Steps to deploy a development version $ git clone https://gitlab.com/mayan-edms/mayan-edms.git $ cd mayan-edms - $ git checkout development + $ git checkout $ virtualenv venv $ source venv/bin/activate $ pip install -r requirements.txt @@ -558,3 +558,10 @@ Manual release :: make release-via-docker-ubuntu + +Other steps +----------- + +#. Update the contrib/scripts/install/docker.sh values + +#. Upload contrib/scripts/install/docker.sh to https://get.mayan-edms.com From 1252263f932fb4f1c9f6118a65754a0a8718c34d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 2 Nov 2019 02:15:05 -0400 Subject: [PATCH 10/10] Add top level config.env file Update the following elements to use the top level config environment file: - CI - Makefile - Docker makefile - Documenation Signed-off-by: Roberto Rosario --- .gitlab-ci.yml | 6 ++- Makefile | 8 ++-- config.env | 9 ++++ docker/Dockerfile | 20 +++++--- docker/Makefile | 7 ++- docs/chapters/backups.rst | 2 +- docs/chapters/deploying.rst | 6 +-- docs/chapters/docker.rst | 6 +-- docs/chapters/scaling_up.rst | 2 +- docs/conf.py | 90 ++++++++++++++++++++++-------------- 10 files changed, 100 insertions(+), 56 deletions(-) create mode 100644 config.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5b56f4358..e9585fa24b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,7 +170,8 @@ test-mysql: - mysql:8.0.3 script: - apt-get install -qq libmysqlclient-dev mysql-client - - pip install mysqlclient + - set -a && . ./config.env && set +a + - pip install mysqlclient==$PYTHON_MYSQL_VERSION - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;" - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations tags: @@ -185,7 +186,8 @@ test-postgres: - postgres script: - apt-get install -qq libpq-dev - - pip install psycopg2 + - set -a && . ./config.env && set +a + - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations tags: - postgres diff --git a/Makefile b/Makefile index 07d5650c5f..f49736c6dd 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +#!make +include config.env .PHONY: clean-pyc clean-build help: @@ -35,7 +37,7 @@ test-launch-postgres: @docker volume rm test-postgres || true docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres sudo apt-get install -q libpq-dev - pip install psycopg2 + pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done test-with-postgres: ## MODULE= - Run tests for a single app, module or test class against a Postgres database container. @@ -55,7 +57,7 @@ test-launch-mysql: @docker volume rm test-mysql || true docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql sudo apt-get install -q libmysqlclient-dev mysql-client - pip install mysqlclient + pip install mysqlclient==$(PYTHON_MYSQL_VERSION) while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done mysql -h 127.0.0.1 -P 3306 -uroot -e "set global character_set_server=utf8mb4;" @@ -77,7 +79,7 @@ test-launch-oracle: @docker volume rm test-oracle || true docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g # https://gist.github.com/kimus/10012910 - pip install cx_Oracle + pip install cx_Oracle==$(PYTHON_ORACLE_VERSION) while ! nc -z 127.0.0.1 49161; do sleep 1; done sleep 10 diff --git a/config.env b/config.env new file mode 100644 index 0000000000..da8df9da35 --- /dev/null +++ b/config.env @@ -0,0 +1,9 @@ +DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6 +DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3 +DOCKER_REDIS_IMAGE_VERSION=redis:5.0 +PYTHON_LIBRABBITMQ_VERSION=1.6.1 +PYTHON_MYSQL_VERSION=1.2.5 +PYTHON_ORACLE_VERSION=7.2.3 +PYTHON_PSYCOPG2_VERSION=2.7.3.2 +PYTHON_PSUTIL_VERSION=5.6.2 +PYTHON_REDIS_VERSION=2.10.6 diff --git a/docker/Dockerfile b/docker/Dockerfile index bd503964c5..8384ac883e 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,6 +8,8 @@ FROM debian:9.8-slim as BASE_IMAGE LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com" +COPY config.env /config.env + ENV PYTHONUNBUFFERED=1 \ LC_ALL=C.UTF-8 \ PROJECT_INSTALL_DIR=/opt/mayan-edms @@ -63,7 +65,7 @@ echo "save \"\"" >> /etc/redis/redis.conf \ && echo "databases 1" >> /etc/redis/redis.conf #### -# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards +# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards #### # Reuse image @@ -104,17 +106,21 @@ apt-get install -y --no-install-recommends \ && chown -R mayan:mayan /src USER mayan -RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \ + +RUN set -a \ +&& . /config.env \ +&& set +a \ +&& python -m virtualenv "${PROJECT_INSTALL_DIR}" \ && . "${PROJECT_INSTALL_DIR}/bin/activate" \ && pip install --no-cache-dir --no-use-pep517 \ - librabbitmq==1.6.1 \ - mysql-python==1.2.5 \ - psycopg2==2.7.3.2 \ - redis==2.10.6 \ + librabbitmq==$PYTHON_LIBRABBITMQ_VERSION \ + mysql-python==$PYTHON_MYSQL_VERSION \ + psycopg2==$PYTHON_PSYCOPG2_VERSION \ + redis==$PYTHON_REDIS_VERSION \ # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \ pip install --no-cache-dir --no-use-pep517 \ - psutil==5.6.2 \ + psutil==$PYTHON_PSUTIL_VERSION \ ; fi \ # Install the Python packages needed to build Mayan EDMS && pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \ diff --git a/docker/Makefile b/docker/Makefile index d3e49fed4b..9a8d9162a0 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,3 +1,6 @@ +#!make +include config.env + APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142 CONSOLE_COLUMNS ?= `echo $$(tput cols)` CONSOLE_LINES ?= `echo $$(tput lines)` @@ -48,13 +51,13 @@ docker-staging-container-postgresql-start: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v mayan-staging-postgres:/var/lib/postgresql/data \ - postgres:9.6-alpine + $(DOCKER_POSTGRES_IMAGE_VERSION) docker-staging-container-redis-start: docker run -d \ --name mayan-staging-redis \ --network=mayan-staging \ - redis:5.0-alpine \ + $(DOCKER_REDIS_IMAGE_VERSION) \ redis-server \ --databases \ "2" \ diff --git a/docs/chapters/backups.rst b/docs/chapters/backups.rst index 5d65284308..496ce9c766 100644 --- a/docs/chapters/backups.rst +++ b/docs/chapters/backups.rst @@ -59,7 +59,7 @@ Example:: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \ - -d postgres:9.6 + |DOCKER_POSTGRES_IMAGE_VERSION| docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump diff --git a/docs/chapters/deploying.rst b/docs/chapters/deploying.rst index 425f3274f9..a14f694f16 100644 --- a/docs/chapters/deploying.rst +++ b/docs/chapters/deploying.rst @@ -84,14 +84,14 @@ For another setup that offers more performance and scalability refer to the ------------------------------------------------------ :: - sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6 + sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION| .. note:: Platforms with the ARM CPU might also need additional requirements. :: - sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2 + sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION| 8. Create the database for the installation: @@ -206,7 +206,7 @@ of a restart or power failure. The Gunicorn workers are increased to 3. ------------------------------------------ :: - sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0 + sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==|PYTHON_LIBRABBITMQ_VERSION| 3. Create the RabbitMQ user and vhost: diff --git a/docs/chapters/docker.rst b/docs/chapters/docker.rst index 9fa58333cf..aca7fbd0f2 100644 --- a/docs/chapters/docker.rst +++ b/docs/chapters/docker.rst @@ -23,7 +23,7 @@ tag here, remember to do so in the next steps also.:: Then download version 9.6 of the Docker PostgreSQL image:: - docker pull postgres:9.6 + docker pull |DOCKER_POSTGRES_IMAGE_VERSION| Create and run a PostgreSQL container:: @@ -35,7 +35,7 @@ Create and run a PostgreSQL container:: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \ - postgres:9.6 + |DOCKER_POSTGRES_IMAGE_VERSION| The PostgreSQL container will have one database named ``mayan``, with an user named ``mayan`` too, with a password of ``mayanuserpass``. The container will @@ -97,7 +97,7 @@ binding (``-p 5432:5432``):: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \ - postgres:9.6 + |DOCKER_POSTGRES_IMAGE_VERSION| Launch the Mayan EDMS container with the network option and change the database hostname to the PostgreSQL container name (``mayan-edms-postgres``) diff --git a/docs/chapters/scaling_up.rst b/docs/chapters/scaling_up.rst index 01a59cecb3..f7eb580164 100644 --- a/docs/chapters/scaling_up.rst +++ b/docs/chapters/scaling_up.rst @@ -92,7 +92,7 @@ section for the required changes. For the Docker image, launch a separate RabbitMQ container (https://hub.docker.com/_/rabbitmq/):: - docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3 + docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan |DOCKER_RABBITMQ_IMAGE_VERSION| Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) to the Mayan EDMS container so that it uses the RabbitMQ container the diff --git a/docs/conf.py b/docs/conf.py index 520ff4791e..5605895f04 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,12 +29,12 @@ sys.path.append( # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] -#extensions = ["djangodocs", "sphinx.ext.intersphinx"] +# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +# extensions = ["djangodocs", "sphinx.ext.intersphinx"] extensions = [ 'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag', 'sphinxcontrib.spelling' @@ -72,20 +72,20 @@ release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True @@ -102,7 +102,7 @@ show_authors = False pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -119,23 +119,23 @@ html_theme_options = { } # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -148,40 +148,40 @@ html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'MayanEDMSdoc' @@ -191,40 +191,42 @@ html_show_sphinx = False # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'MayanEDMS.tex', 'Mayan EDMS Documentation', - mayan.__author__, 'manual'), + ( + 'index', 'MayanEDMS.tex', 'Mayan EDMS Documentation', + mayan.__author__, 'manual' + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -255,5 +257,25 @@ extlinks = { } +def _load_env_file(filename='../config.env'): + result = [] + with open(filename) as file_object: + for line in file_object: + if not line.startswith('#'): + key, value = line.strip().split('=') + result.append(('|{}|'.format(key), value)) + + return result + + +def GlobalSubstitution(app, docname, source): + for old, new in global_subtitutions: + source[0] = source[0].replace(old, new) + + def setup(app): app.add_stylesheet('css/custom.css') + app.connect('source-read', GlobalSubstitution) + + +global_subtitutions = _load_env_file()