From c665e758714d38485fd6320e2ace1f09ef56b8e7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 1 Aug 2018 05:09:43 -0400 Subject: [PATCH 01/18] Improve serialization migration for the models: Document, DocumentVersion, DocumentMetadata and DocumentTypeOCRSettings Signed-off-by: Roberto Rosario --- mayan/apps/documents/managers.py | 15 ++++++++++++++- mayan/apps/documents/models.py | 11 +++++++++-- mayan/apps/metadata/managers.py | 11 +++++++++++ mayan/apps/metadata/models.py | 6 +++++- mayan/apps/ocr/managers.py | 13 +++++++++++++ mayan/apps/ocr/models.py | 11 ++++++++++- 6 files changed, 62 insertions(+), 5 deletions(-) diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index a783670af0..d6d3dc7946 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -20,7 +20,7 @@ class DocumentManager(models.Manager): stale_stub_document.delete(trash=False) def get_by_natural_key(self, uuid): - return self.get(uuid=uuid) + return self.model.passthrough.get(uuid=uuid) def get_queryset(self): return TrashCanQuerySet( @@ -32,6 +32,19 @@ class DocumentManager(models.Manager): document.invalidate_cache() +class DocumentVersionManager(models.Manager): + def get_by_natural_key(self, checksum, document_natural_key): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + try: + document = Document.objects.get_by_natural_key(document_natural_key) + except Document.DoesNotExist: + raise self.model.DoesNotExist + + return document.versions.get(checksum=checksum) + + class DocumentTypeManager(models.Manager): def check_delete_periods(self): logger.info('Executing') diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index afdaeb4e06..f98afd2b5f 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -33,8 +33,9 @@ from .events import ( ) from .literals import DEFAULT_DELETE_PERIOD, DEFAULT_DELETE_TIME_UNIT from .managers import ( - DocumentManager, DocumentTypeManager, DuplicatedDocumentManager, - PassthroughManager, RecentDocumentManager, TrashCanManager + DocumentManager, DocumentVersionManager, DocumentTypeManager, + DuplicatedDocumentManager, PassthroughManager, RecentDocumentManager, + TrashCanManager ) from .permissions import permission_document_view from .settings import ( @@ -415,6 +416,8 @@ class DocumentVersion(models.Model): verbose_name = _('Document version') verbose_name_plural = _('Document version') + objects = DocumentVersionManager() + def __str__(self): return self.get_rendered_string() @@ -505,6 +508,10 @@ class DocumentVersion(models.Model): context=Context({'instance': self}) ) + def natural_key(self): + return (self.checksum,) + self.document.natural_key() + natural_key.dependencies = ['documents.Document'] + def invalidate_cache(self): storage_documentimagecache.delete(self.cache_filename) for page in self.pages.all(): diff --git a/mayan/apps/metadata/managers.py b/mayan/apps/metadata/managers.py index 68775323ab..2a07fc459a 100644 --- a/mayan/apps/metadata/managers.py +++ b/mayan/apps/metadata/managers.py @@ -24,6 +24,17 @@ class MetadataTypeManager(models.Manager): class DocumentTypeMetadataTypeManager(models.Manager): + def get_by_natural_key(self, metadata_type_name, document_natural_key): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + try: + document = Document.objects.get_by_natural_key(document_natural_key) + except Document.DoesNotExist: + raise self.model.DoesNotExist + + return document.metadata.get(metadata_type_name=metadata_type_name) + def get_metadata_types_for(self, document_type): DocumentType = apps.get_model( app_label='metadata', model_name='MetadataType' diff --git a/mayan/apps/metadata/models.py b/mayan/apps/metadata/models.py index b7e5a21503..89d312d1b0 100644 --- a/mayan/apps/metadata/models.py +++ b/mayan/apps/metadata/models.py @@ -111,7 +111,7 @@ class MetadataType(models.Model): ).exists() def natural_key(self): - return (self.name,) + return self.name def validate_value(self, document_type, value): # Check default @@ -189,6 +189,10 @@ class DocumentMetadata(models.Model): return super(DocumentMetadata, self).delete(*args, **kwargs) + def natural_key(self): + return (self.metadata_type.name,) + self.document.natural_key() + natural_key.dependencies = ['documents.Document', 'metadata.MetadataType'] + @property def is_required(self): return self.metadata_type.get_required_for( diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index fb547a3648..20de7b30b6 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -90,3 +90,16 @@ class DocumentPageOCRContentManager(models.Manager): post_document_version_ocr.send( sender=document_version.__class__, instance=document_version ) + + +class DocumentTypeSettingsManager(models.Manager): + def get_by_natural_key(self, document_type_natural_key): + DocumentType = apps.get_model( + app_label='documents', model_name='DocumentType' + ) + try: + document_type = DocumentType.objects.get_by_natural_key(document_type_natural_key) + except DocumentType.DoesNotExist: + raise self.model.DoesNotExist + + return document_type.ocr_settings diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index 0d0b099627..293d6ff82b 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -6,7 +6,9 @@ from django.utils.translation import ugettext_lazy as _ from documents.models import DocumentPage, DocumentType, DocumentVersion -from .managers import DocumentPageOCRContentManager +from .managers import ( + DocumentPageOCRContentManager, DocumentTypeSettingsManager +) class DocumentTypeSettings(models.Model): @@ -22,10 +24,17 @@ class DocumentTypeSettings(models.Model): verbose_name=_('Automatically queue newly created documents for OCR.') ) + objects = DocumentTypeSettingsManager() + class Meta: verbose_name = _('Document type settings') verbose_name_plural = _('Document types settings') + def natural_key(self): + return self.document_type.natural_key() + natural_key.dependencies = ['documents.DocumentType'] + + @python_2_unicode_compatible class DocumentPageOCRContent(models.Model): From 1ad7d2a4064477e4c171c5cc19ef2d7bdefa167d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 2 Aug 2018 15:56:47 -0400 Subject: [PATCH 02/18] Limit the number of branches that trigger the full test suit. Signed-off-by: Roberto Rosario --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c2ea509d0..90da9d8733 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,11 @@ job_docker_nightly: - export LC_ALL=en_US.UTF-8 - apt-get install -qq curl gcc ghostscript gpgv gnupg graphviz libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu - pip install -r requirements/testing.txt + only: + - master + - versions/next + - nightly + - staging test-mysql: <<: *test_base From e18c043c1fd7c9e3ffeaf10d76466b53e3873c75 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 2 Aug 2018 16:00:22 -0400 Subject: [PATCH 03/18] Improve natural key handing for the Document, Metadata, DocumentMetadata, DocumentTypeOCRSetting and UserProfileLocale models. Signed-off-by: Roberto Rosario --- mayan/apps/common/managers.py | 12 ++++++++++++ mayan/apps/common/models.py | 9 ++++++++- mayan/apps/documents/managers.py | 5 +++-- mayan/apps/documents/models.py | 2 +- mayan/apps/metadata/managers.py | 12 ++++++++++-- mayan/apps/metadata/models.py | 4 ++-- mayan/apps/ocr/managers.py | 2 +- 7 files changed, 37 insertions(+), 9 deletions(-) diff --git a/mayan/apps/common/managers.py b/mayan/apps/common/managers.py index c99c8f631d..9dbcbe97d4 100644 --- a/mayan/apps/common/managers.py +++ b/mayan/apps/common/managers.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.apps import apps +from django.contrib.auth import get_user_model from django.contrib.contenttypes.fields import GenericRelation from django.db import models @@ -11,3 +12,14 @@ class ErrorLogEntryManager(models.Manager): app_label='common', model_name='ErrorLogEntry' ) model.add_to_class('error_logs', GenericRelation(ErrorLogEntry)) + + +class UserLocaleProfileManager(models.Manager): + def get_by_natural_key(self, user_natural_key): + User = get_user_model() + try: + user = User.objects.get_by_natural_key(user_natural_key) + except User.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(user__pk=user.pk) diff --git a/mayan/apps/common/models.py b/mayan/apps/common/models.py index 298f1dba0b..2bcc9232c0 100644 --- a/mayan/apps/common/models.py +++ b/mayan/apps/common/models.py @@ -11,7 +11,7 @@ from django.db import models from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from .managers import ErrorLogEntryManager +from .managers import ErrorLogEntryManager, UserLocaleProfileManager from .storages import storage_sharedupload @@ -88,9 +88,16 @@ class UserLocaleProfile(models.Model): choices=settings.LANGUAGES, max_length=8, verbose_name=_('Language') ) + objects = UserLocaleProfileManager() + class Meta: verbose_name = _('User locale profile') verbose_name_plural = _('User locale profiles') def __str__(self): return force_text(self.user) + + def natural_key(self): + return self.user.natural_key() + natural_key.dependencies = [settings.AUTH_USER_MODEL] + diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index d6d3dc7946..5efb8d9f80 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -6,6 +6,7 @@ import logging from django.apps import apps from django.db import models from django.db.models import F, Max +from django.utils.encoding import force_text from django.utils.timezone import now from .literals import STUB_EXPIRATION_INTERVAL @@ -20,7 +21,7 @@ class DocumentManager(models.Manager): stale_stub_document.delete(trash=False) def get_by_natural_key(self, uuid): - return self.model.passthrough.get(uuid=uuid) + return self.model.passthrough.get(uuid=force_text(uuid)) def get_queryset(self): return TrashCanQuerySet( @@ -33,7 +34,7 @@ class DocumentManager(models.Manager): class DocumentVersionManager(models.Manager): - def get_by_natural_key(self, checksum, document_natural_key): + def get_by_natural_key(self, document_natural_key, checksum): Document = apps.get_model( app_label='documents', model_name='Document' ) diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index f98afd2b5f..370cbd0ada 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -509,7 +509,7 @@ class DocumentVersion(models.Model): ) def natural_key(self): - return (self.checksum,) + self.document.natural_key() + return self.document.natural_key() + (self.checksum,) natural_key.dependencies = ['documents.Document'] def invalidate_cache(self): diff --git a/mayan/apps/metadata/managers.py b/mayan/apps/metadata/managers.py index 2a07fc459a..76f8830d19 100644 --- a/mayan/apps/metadata/managers.py +++ b/mayan/apps/metadata/managers.py @@ -24,16 +24,24 @@ class MetadataTypeManager(models.Manager): class DocumentTypeMetadataTypeManager(models.Manager): - def get_by_natural_key(self, metadata_type_name, document_natural_key): + def get_by_natural_key(self, document_natural_key, metadata_type_natural_key): Document = apps.get_model( app_label='documents', model_name='Document' ) + MetadataType = apps.get_model( + app_label='metadata', model_name='MetadataType' + ) try: document = Document.objects.get_by_natural_key(document_natural_key) except Document.DoesNotExist: raise self.model.DoesNotExist + else: + try: + metadata_type = MetadataType.objects.get_by_natural_key(metadata_type_natural_key) + except MetadataType.DoesNotExist: + raise self.model.DoesNotExist - return document.metadata.get(metadata_type_name=metadata_type_name) + return self.get(document__pk=document.pk, metadata_type__pk=metadata_type.pk) def get_metadata_types_for(self, document_type): DocumentType = apps.get_model( diff --git a/mayan/apps/metadata/models.py b/mayan/apps/metadata/models.py index 89d312d1b0..be67f0093b 100644 --- a/mayan/apps/metadata/models.py +++ b/mayan/apps/metadata/models.py @@ -111,7 +111,7 @@ class MetadataType(models.Model): ).exists() def natural_key(self): - return self.name + return (self.name,) def validate_value(self, document_type, value): # Check default @@ -190,7 +190,7 @@ class DocumentMetadata(models.Model): return super(DocumentMetadata, self).delete(*args, **kwargs) def natural_key(self): - return (self.metadata_type.name,) + self.document.natural_key() + return self.document.natural_key() + self.metadata_type.natural_key() natural_key.dependencies = ['documents.Document', 'metadata.MetadataType'] @property diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index 20de7b30b6..c6723e1926 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -102,4 +102,4 @@ class DocumentTypeSettingsManager(models.Manager): except DocumentType.DoesNotExist: raise self.model.DoesNotExist - return document_type.ocr_settings + return self.get(document_type__pk=document_type.pk) From fc304394f5501c6e326a67f768fbaf072c7a436b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 18:50:15 -0400 Subject: [PATCH 04/18] Add convertdb command. Signed-off-by: Roberto Rosario --- HISTORY.rst | 7 +- .../common/management/commands/convertdb.py | 86 +++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 mayan/apps/common/management/commands/convertdb.py diff --git a/HISTORY.rst b/HISTORY.rst index 2972f4b45d..c9567eaeb6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,4 +1,9 @@ -3.0.1 (208-07-08) +3.1 (2018-XX-XX) +================ +- Improve database vendor migration support +- Add convertdb management command. + +3.0.1 (2018-07-08) ================= - Pin javascript libraries to specific versions to avoid using potentianlly broken updates automatically. GitLab issue #486. diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py new file mode 100644 index 0000000000..b2d4f19f6f --- /dev/null +++ b/mayan/apps/common/management/commands/convertdb.py @@ -0,0 +1,86 @@ +from __future__ import unicode_literals + +import errno +import os + +from pathlib import Path + +from django.conf import settings +from django.core import management +from django.core.management.base import CommandError +from django.utils.encoding import force_text +from django.utils.translation import ugettext_lazy as _ + +from common.utils import fs_cleanup +from documents.models import DocumentType + +CONVERTDB_FOLDER = 'convertdb' +CONVERTDB_OUTPUT_FILENAME = 'migrate.json' + + +class Command(management.BaseCommand): + help = 'Convert from a database backend to another one.' + + def add_arguments(self, parser): + parser.add_argument( + '--from', action='store', default='default', dest='from', + help=_( + 'The database from which data will be exported. If omitted ' + 'the database named "default" will be used.' + ), + ) + parser.add_argument( + '--to', action='store', default='default', dest='to', + help=_( + 'The database to which data will be imported. If omitted ' + 'the database named "default" will be used.' + ), + ) + parser.add_argument( + '--force', action='store_true', dest='force', + help=_( + 'Force the conversion of the database even if the receving ' + 'database is not empty.' + ), + ) + + def handle(self, *args, **options): + # Create the media/convertdb folder + convertdb_folder_path = force_text( + Path( + settings.MEDIA_ROOT, CONVERTDB_FOLDER + ) + ) + + try: + os.makedirs(convertdb_folder_path) + except OSError as exception: + if exception.errno == errno.EEXIST: + pass + + convertdb_file_path = force_text( + Path( + convertdb_folder_path, CONVERTDB_OUTPUT_FILENAME + ) + ) + + management.call_command( + 'dumpdata', all=True, database=options['from'], + natural_primary=True, natural_foreign=True, + output=convertdb_file_path, interactive=False, + indent=4, + ) + + if DocumentType.objects.using('default').count() and not options['force']: + fs_cleanup(convertdb_file_path) + raise CommandError( + 'There is existing data in the database that will be ' + 'used for the import. If you proceed with the conversion ' + 'you might lose data. Please check you settings.' + ) + + management.call_command( + 'loaddata', convertdb_file_path, database=options['to'], + interactive=False + ) + fs_cleanup(convertdb_file_path) From 4dea5911da6b13b9df28cf6bbddd155517fca2fb Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 18:50:40 -0400 Subject: [PATCH 05/18] Redirect to the cabinet list view after creating a new cabinet. Signed-off-by: Roberto Rosario --- mayan/apps/cabinets/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mayan/apps/cabinets/views.py b/mayan/apps/cabinets/views.py index 95e08bdfcb..7466b27ba2 100644 --- a/mayan/apps/cabinets/views.py +++ b/mayan/apps/cabinets/views.py @@ -31,6 +31,7 @@ logger = logging.getLogger(__name__) class CabinetCreateView(SingleObjectCreateView): fields = ('label',) model = Cabinet + post_action_redirect = reverse_lazy('cabinets:cabinet_list') view_permission = permission_cabinet_create def get_extra_context(self): From f42e1a96b28b5e93ffc9605c1e2b11fd8e38ec21 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 18:51:17 -0400 Subject: [PATCH 06/18] Add natural key support to the Index model. Signed-off-by: Roberto Rosario --- mayan/apps/document_indexing/managers.py | 4 ++-- mayan/apps/document_indexing/models.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mayan/apps/document_indexing/managers.py b/mayan/apps/document_indexing/managers.py index fe0477768b..8aa1f5478e 100644 --- a/mayan/apps/document_indexing/managers.py +++ b/mayan/apps/document_indexing/managers.py @@ -9,8 +9,8 @@ class DocumentIndexInstanceNodeManager(models.Manager): class IndexManager(models.Manager): - def get_by_natural_key(self, name): - return self.get(name=name) + def get_by_natural_key(self, slug): + return self.get(slug=slug) def index_document(self, document): for index in self.filter(enabled=True, document_types=document.document_type): diff --git a/mayan/apps/document_indexing/models.py b/mayan/apps/document_indexing/models.py index 78ef2ff4bc..5012a5e6ff 100644 --- a/mayan/apps/document_indexing/models.py +++ b/mayan/apps/document_indexing/models.py @@ -83,6 +83,9 @@ class Index(models.Model): def instance_root(self): return self.template_root.index_instance_nodes.get() + def natural_key(self): + return (self.slug,) + def rebuild(self): """ Delete and reconstruct the index by deleting of all its instance nodes From 06265292283185527bb5401f42ad099afd2b13bc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 18:52:13 -0400 Subject: [PATCH 07/18] Add natural key support to Document pages. Signed-off-by: Roberto Rosario --- mayan/apps/documents/managers.py | 40 +++++++++++++++++++++++++++++--- mayan/apps/documents/models.py | 16 +++++++++---- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 5efb8d9f80..5a835ec83c 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -4,6 +4,7 @@ from datetime import timedelta import logging from django.apps import apps +from django.contrib.auth import get_user_model from django.db import models from django.db.models import F, Max from django.utils.encoding import force_text @@ -33,17 +34,30 @@ class DocumentManager(models.Manager): document.invalidate_cache() +class DocumentPageManager(models.Manager): + def get_by_natural_key(self, page_number, document_version_natural_key): + DocumentVersion = apps.get_model( + app_label='documents', model_name='DocumentVersion' + ) + try: + document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) + except DocumentVersion.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document_version__pk=document_version.pk, page_number=page_number) + + class DocumentVersionManager(models.Manager): - def get_by_natural_key(self, document_natural_key, checksum): + def get_by_natural_key(self, checksum, document_natural_key): Document = apps.get_model( app_label='documents', model_name='Document' ) try: - document = Document.objects.get_by_natural_key(document_natural_key) + document = Document.objects.get_by_natural_key(*document_natural_key) except Document.DoesNotExist: raise self.model.DoesNotExist - return document.versions.get(checksum=checksum) + return self.get(document__pk=document.pk, checksum=checksum) class DocumentTypeManager(models.Manager): @@ -189,6 +203,26 @@ class RecentDocumentManager(models.Manager): self.filter(pk__in=list(recent_to_delete)).delete() return new_recent + def get_by_natural_key(self, datetime_accessed, document_natural_key, user_natural_key): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + User = get_user_model() + try: + document = Document.objects.get_by_natural_key(*document_natural_key) + except Document.DoesNotExist: + raise self.model.DoesNotExist + else: + try: + user = User.objects.get_by_natural_key(*user_natural_key) + except User.DoesNotExist: + raise self.model.DoesNotExist + + return self.get( + document__pk=document.pk, user__pk=user.pk, + datetime_accessed=datetime_accessed + ) + def get_for_user(self, user): document_model = apps.get_model('documents', 'document') diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 370cbd0ada..aeb8791e9a 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -33,9 +33,9 @@ from .events import ( ) from .literals import DEFAULT_DELETE_PERIOD, DEFAULT_DELETE_TIME_UNIT from .managers import ( - DocumentManager, DocumentVersionManager, DocumentTypeManager, - DuplicatedDocumentManager, PassthroughManager, RecentDocumentManager, - TrashCanManager + DocumentManager, DocumentPageManager, DocumentVersionManager, + DocumentTypeManager, DuplicatedDocumentManager, PassthroughManager, + RecentDocumentManager, TrashCanManager ) from .permissions import permission_document_view from .settings import ( @@ -509,7 +509,7 @@ class DocumentVersion(models.Model): ) def natural_key(self): - return self.document.natural_key() + (self.checksum,) + return (self.checksum, self.document.natural_key()) natural_key.dependencies = ['documents.Document'] def invalidate_cache(self): @@ -735,6 +735,8 @@ class DocumentPage(models.Model): verbose_name=_('Page number') ) + objects = DocumentPageManager() + class Meta: ordering = ('page_number',) verbose_name = _('Document page') @@ -894,6 +896,10 @@ class DocumentPage(models.Model): for cached_image in self.cached_images.all(): cached_image.delete() + def natural_key(self): + return (self.page_number, self.document_version.natural_key()) + natural_key.dependencies = ['documents.DocumentVersion'] + @property def siblings(self): return DocumentPage.objects.filter( @@ -962,7 +968,7 @@ class RecentDocument(models.Model): return force_text(self.document) def natural_key(self): - return self.document.natural_key() + self.user.natural_key() + return (self.datetime_accessed, self.document.natural_key(), self.user.natural_key()) natural_key.dependencies = ['documents.Document', settings.AUTH_USER_MODEL] From cbcb8a84c5dbd3da8c762ecb5e42db5cbf380d69 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 18:52:53 -0400 Subject: [PATCH 08/18] Add natural key support to the user mailer model. Signed-off-by: Roberto Rosario --- mayan/apps/mailer/managers.py | 8 ++++++++ mayan/apps/mailer/models.py | 6 ++++++ 2 files changed, 14 insertions(+) create mode 100644 mayan/apps/mailer/managers.py diff --git a/mayan/apps/mailer/managers.py b/mayan/apps/mailer/managers.py new file mode 100644 index 0000000000..4933885211 --- /dev/null +++ b/mayan/apps/mailer/managers.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +from django.db import models + + +class UserMailerManager(models.Manager): + def get_by_natural_key(self, label): + return self.get(label=label) diff --git a/mayan/apps/mailer/models.py b/mayan/apps/mailer/models.py index c186bade64..d51fac223d 100644 --- a/mayan/apps/mailer/models.py +++ b/mayan/apps/mailer/models.py @@ -11,6 +11,7 @@ from django.utils.html import strip_tags from django.utils.module_loading import import_string from django.utils.translation import ugettext_lazy as _ +from .managers import UserMailerManager from .utils import split_recipient_list logger = logging.getLogger(__name__) @@ -51,6 +52,8 @@ class UserMailer(models.Model): blank=True, verbose_name=_('Backend data') ) + objects = UserMailerManager() + class Meta: ordering = ('label',) verbose_name = _('User mailer') @@ -77,6 +80,9 @@ class UserMailer(models.Model): def loads(self): return json.loads(self.backend_data) + def natural_key(self): + return (self.label,) + def save(self, *args, **kwargs): if self.default: UserMailer.objects.select_for_update().exclude(pk=self.pk).update( From f756584176f51554a649ac3ba10d150c0133893c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 18:54:30 -0400 Subject: [PATCH 09/18] Make source label field unique. Signed-off-by: Roberto Rosario --- .../migrations/0019_auto_20180803_0440.py | 20 +++++++++++++++++++ mayan/apps/sources/models.py | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 mayan/apps/sources/migrations/0019_auto_20180803_0440.py diff --git a/mayan/apps/sources/migrations/0019_auto_20180803_0440.py b/mayan/apps/sources/migrations/0019_auto_20180803_0440.py new file mode 100644 index 0000000000..4686d5107a --- /dev/null +++ b/mayan/apps/sources/migrations/0019_auto_20180803_0440.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-08-03 04:40 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0018_auto_20180608_0057'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='label', + field=models.CharField(db_index=True, max_length=64, unique=True, verbose_name='Label'), + ), + ] diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 5d37262576..60878a0f04 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -51,7 +51,9 @@ logger = logging.getLogger(__name__) @python_2_unicode_compatible class Source(models.Model): - label = models.CharField(max_length=64, verbose_name=_('Label')) + label = models.CharField( + db_index=True, max_length=64, unique=True, verbose_name=_('Label') + ) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) objects = InheritanceManager() From 0c1c38917c8cddb550585c602a83beb5661e36e1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 19:25:24 -0400 Subject: [PATCH 10/18] Make format explicit JSON Signed-off-by: Roberto Rosario --- mayan/apps/common/management/commands/convertdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py index b2d4f19f6f..3086ce0e21 100644 --- a/mayan/apps/common/management/commands/convertdb.py +++ b/mayan/apps/common/management/commands/convertdb.py @@ -68,7 +68,7 @@ class Command(management.BaseCommand): 'dumpdata', all=True, database=options['from'], natural_primary=True, natural_foreign=True, output=convertdb_file_path, interactive=False, - indent=4, + format='json' ) if DocumentType.objects.using('default').count() and not options['force']: From b14d7d6da0b08d8f316f5dcd21fe397416bb2a6f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Aug 2018 19:25:49 -0400 Subject: [PATCH 11/18] Add release notes for version 3.1. Signed-off-by: Roberto Rosario --- docs/releases/3.1.rst | 68 +++++++++++++++++++++++++++++++++++++++++ docs/releases/index.rst | 1 + 2 files changed, 69 insertions(+) create mode 100644 docs/releases/3.1.rst diff --git a/docs/releases/3.1.rst b/docs/releases/3.1.rst new file mode 100644 index 0000000000..4f346516cf --- /dev/null +++ b/docs/releases/3.1.rst @@ -0,0 +1,68 @@ +============================= +Mayan EDMS v3.1 release notes +============================= + +Released: XX, 2018 + +What's new +========== + +Removals +-------- +- None + +Upgrading from a previous version +--------------------------------- + + +Using PIP +~~~~~~~~~ + +Type in the console:: + + $ pip install mayan-edms==3.0.1 + +the requirements will also be updated automatically. + + +Using Git +~~~~~~~~~ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +~~~~~~~~~~~~ + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py collectstatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +============================= + +* None + +Bugs fixed or issues closed +=========================== + +* `GitLab issue #486 `_ Docker Verison 3.0 not working + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index e304e09d5e..4c50d95b70 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -22,6 +22,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.1 3.0.1 3.0 From 10231363e678f0b183ec75efbe24e81dd6f97ec9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 5 Aug 2018 20:07:57 -0400 Subject: [PATCH 12/18] Add natural key support to the document parsing app. Signed-off-by: Roberto Rosario --- mayan/apps/document_parsing/managers.py | 14 ++++++++++++++ mayan/apps/document_parsing/models.py | 8 +++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mayan/apps/document_parsing/managers.py b/mayan/apps/document_parsing/managers.py index f5d63bdda3..c435114b41 100644 --- a/mayan/apps/document_parsing/managers.py +++ b/mayan/apps/document_parsing/managers.py @@ -4,6 +4,7 @@ import logging import sys import traceback +from django.apps import apps from django.conf import settings from django.db import models @@ -48,3 +49,16 @@ class DocumentPageContentManager(models.Manager): action_object=document_version.document, target=document_version ) + + +class DocumentTypeSettingsManager(models.Manager): + def get_by_natural_key(self, document_type_natural_key): + DocumentType = apps.get_model( + app_label='documents', model_name='DocumentType' + ) + try: + document_type = DocumentType.objects.get_by_natural_key(document_type_natural_key) + except DocumentType.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document_type__pk=document_type.pk) diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 30ea3ee81f..c17caf39c1 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _ from documents.models import DocumentPage, DocumentType, DocumentVersion -from .managers import DocumentPageContentManager +from .managers import DocumentPageContentManager, DocumentTypeSettingsManager @python_2_unicode_compatible @@ -37,6 +37,12 @@ class DocumentTypeSettings(models.Model): verbose_name=_('Automatically queue newly created documents for parsing.') ) + objects = DocumentTypeSettingsManager() + + def natural_key(self): + return self.document_type.natural_key() + natural_key.dependencies = ['documents.DocumentType'] + class Meta: verbose_name = _('Document type settings') verbose_name_plural = _('Document types settings') From 9183369bdb959ffed049d9f1d05e40f79ad903c1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 5 Aug 2018 20:44:37 -0400 Subject: [PATCH 13/18] Use patched version of django-celery with support for natural key. https://github.com/celery/django-celery/pull/552 Signed-off-by: Roberto Rosario --- requirements/base.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index bb9bd0b986..83c20c89b9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,7 +6,8 @@ cssmin==0.2.0 django-activity-stream==0.6.5 django-autoadmin==1.1.1 -django-celery==3.2.1 +#django-celery==3.2.1 - Use fork below until patch https://github.com/celery/django-celery/pull/552 is accepted. +https://github.com/mayan-edms/django-celery/zipball/master#egg=django-celery django-colorful==1.2 django-compressor==2.2 django-cors-headers==2.2.0 From c02a8bb3e34dc398ac13ae60495353f40775199e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 6 Aug 2018 14:43:30 -0400 Subject: [PATCH 14/18] Switch to pathlib2 Signed-off-by: Roberto Rosario --- removals.txt | 1 + requirements/base.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/removals.txt b/removals.txt index 5c63293032..8683d4f893 100644 --- a/removals.txt +++ b/removals.txt @@ -1,5 +1,6 @@ # Packages to be remove during upgrades django-filetransfers django-rest-swagger +pathlib pytesseract pdfminer diff --git a/requirements/base.txt b/requirements/base.txt index 83c20c89b9..ef06bde62c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -39,7 +39,7 @@ mock==2.0.0 node-semver==0.3.0 -pathlib==1.0.1 +pathlib2==2.3.2 pycountry==18.5.26 PyPDF2==1.26.0 pyocr==0.5.1 From 7994803e8f8b1ad26071d97852543176aae26ef4 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 6 Aug 2018 14:44:03 -0400 Subject: [PATCH 15/18] Add the upstream version of django-celery to the removal list. Signed-off-by: Roberto Rosario --- removals.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/removals.txt b/removals.txt index 8683d4f893..0b87897201 100644 --- a/removals.txt +++ b/removals.txt @@ -1,4 +1,5 @@ # Packages to be remove during upgrades +django-celery django-filetransfers django-rest-swagger pathlib From 3937696774cba4f3714944b4a51d35621ad98f2c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 6 Aug 2018 14:54:55 -0400 Subject: [PATCH 16/18] Add natural key support to the models of the checkouts app. Signed-off-by: Roberto Rosario --- mayan/apps/checkouts/managers.py | 23 +++++++++++++++++++++++ mayan/apps/checkouts/models.py | 8 ++++++++ 2 files changed, 31 insertions(+) diff --git a/mayan/apps/checkouts/managers.py b/mayan/apps/checkouts/managers.py index 6d795729ce..a42fa7ba95 100644 --- a/mayan/apps/checkouts/managers.py +++ b/mayan/apps/checkouts/managers.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, unicode_literals import logging +from django.apps import apps from django.db import models from django.utils.timezone import now @@ -88,6 +89,17 @@ class DocumentCheckoutManager(models.Manager): else: return not checkout_info.block_new_version + def get_by_natural_key(self, document_natural_key): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + try: + document = Document.objects.get_by_natural_key(document_natural_key) + except Document.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document__pk=document.pk) + class NewVersionBlockManager(models.Manager): def block(self, document): @@ -98,3 +110,14 @@ class NewVersionBlockManager(models.Manager): def is_blocked(self, document): return self.filter(document=document).exists() + + def get_by_natural_key(self, document_natural_key): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + try: + document = Document.objects.get_by_natural_key(document_natural_key) + except Document.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document__pk=document.pk) diff --git a/mayan/apps/checkouts/models.py b/mayan/apps/checkouts/models.py index 2750fdf85f..43a38d8848 100644 --- a/mayan/apps/checkouts/models.py +++ b/mayan/apps/checkouts/models.py @@ -72,6 +72,10 @@ class DocumentCheckout(models.Model): def get_absolute_url(self): return reverse('checkout:checkout_info', args=(self.document.pk,)) + def natural_key(self): + return self.document.natural_key() + natural_key.dependencies = ['documents.Document'] + def save(self, *args, **kwargs): # TODO: enclose in transaction new_checkout = not self.pk @@ -104,3 +108,7 @@ class NewVersionBlock(models.Model): class Meta: verbose_name = _('New version block') verbose_name_plural = _('New version blocks') + + def natural_key(self): + return self.document.natural_key() + natural_key.dependencies = ['documents.Document'] From 14bea940304da1d05b7116916962fa68456b2737 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 6 Aug 2018 14:55:14 -0400 Subject: [PATCH 17/18] Update pathlib imports to pathlib2. Signed-off-by: Roberto Rosario --- mayan/apps/common/javascript.py | 2 +- mayan/apps/common/management/commands/convertdb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mayan/apps/common/javascript.py b/mayan/apps/common/javascript.py index c3cfdbbda4..c772f3fb38 100644 --- a/mayan/apps/common/javascript.py +++ b/mayan/apps/common/javascript.py @@ -8,7 +8,7 @@ import shutil import tarfile from furl import furl -from pathlib import Path +from pathlib2 import Path import requests from semver import max_satisfying diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py index 3086ce0e21..f78226f0d6 100644 --- a/mayan/apps/common/management/commands/convertdb.py +++ b/mayan/apps/common/management/commands/convertdb.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import errno import os -from pathlib import Path +from pathlib2 import Path from django.conf import settings from django.core import management From 493ba214f2f5d27b7181e2ec1ef4c8e98bed4e52 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 6 Aug 2018 14:57:28 -0400 Subject: [PATCH 18/18] Sort the manager methods of the checkouts app. Signed-off-by: Roberto Rosario --- mayan/apps/checkouts/managers.py | 78 ++++++++++++++++---------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/mayan/apps/checkouts/managers.py b/mayan/apps/checkouts/managers.py index a42fa7ba95..cdd4946fa4 100644 --- a/mayan/apps/checkouts/managers.py +++ b/mayan/apps/checkouts/managers.py @@ -19,37 +19,13 @@ logger = logging.getLogger(__name__) class DocumentCheckoutManager(models.Manager): - def checkout_document(self, document, expiration_datetime, user, block_new_version=True): - return self.create( - document=document, expiration_datetime=expiration_datetime, - user=user, block_new_version=block_new_version - ) - - def checked_out_documents(self): - return Document.objects.filter( - pk__in=self.model.objects.all().values_list( - 'document__pk', flat=True - ) - ) - - def expired_check_outs(self): - expired_list = Document.objects.filter( - pk__in=self.model.objects.filter( - expiration_datetime__lte=now() - ).values_list('document__pk', flat=True) - ) - logger.debug('expired_list: %s', expired_list) - return expired_list - - def check_in_expired_check_outs(self): - for document in self.expired_check_outs(): - document.check_in() - - def is_document_checked_out(self, document): - if self.model.objects.filter(document=document): + def are_document_new_versions_allowed(self, document, user=None): + try: + checkout_info = self.document_checkout_info(document) + except DocumentNotCheckedOut: return True else: - return False + return not checkout_info.block_new_version def check_in_document(self, document, user=None): try: @@ -69,6 +45,23 @@ class DocumentCheckoutManager(models.Manager): document_checkout.delete() + def check_in_expired_check_outs(self): + for document in self.expired_check_outs(): + document.check_in() + + def checkout_document(self, document, expiration_datetime, user, block_new_version=True): + return self.create( + document=document, expiration_datetime=expiration_datetime, + user=user, block_new_version=block_new_version + ) + + def checked_out_documents(self): + return Document.objects.filter( + pk__in=self.model.objects.all().values_list( + 'document__pk', flat=True + ) + ) + def document_checkout_info(self, document): try: return self.model.objects.get(document=document) @@ -81,13 +74,14 @@ class DocumentCheckoutManager(models.Manager): else: return STATE_CHECKED_IN - def are_document_new_versions_allowed(self, document, user=None): - try: - checkout_info = self.document_checkout_info(document) - except DocumentNotCheckedOut: - return True - else: - return not checkout_info.block_new_version + def expired_check_outs(self): + expired_list = Document.objects.filter( + pk__in=self.model.objects.filter( + expiration_datetime__lte=now() + ).values_list('document__pk', flat=True) + ) + logger.debug('expired_list: %s', expired_list) + return expired_list def get_by_natural_key(self, document_natural_key): Document = apps.get_model( @@ -100,14 +94,17 @@ class DocumentCheckoutManager(models.Manager): return self.get(document__pk=document.pk) + def is_document_checked_out(self, document): + if self.model.objects.filter(document=document): + return True + else: + return False + class NewVersionBlockManager(models.Manager): def block(self, document): self.get_or_create(document=document) - def unblock(self, document): - self.filter(document=document).delete() - def is_blocked(self, document): return self.filter(document=document).exists() @@ -121,3 +118,6 @@ class NewVersionBlockManager(models.Manager): raise self.model.DoesNotExist return self.get(document__pk=document.pk) + + def unblock(self, document): + self.filter(document=document).delete()