Merge branch 'feature/db_migration' into versions/next

HISTORY.rst

@@ -1,5 +1,7 @@
 3.1 (2018-XX-XX)
 ================
+- Improve database vendor migration support
+- Add convertdb management command.
 - Fix crop transformation argument parsing. Thanks to Jordan Wages
   (@wagesj45). Closes GitLab issue #490
 - Add error checking to the crop transformation arguments.

docs/releases/3.1.rst

@@ -0,0 +1,68 @@
+=============================
+Mayan EDMS v3.1 release notes
+=============================
+
+Released: XX, 2018
+
+What's new
+==========
+
+Removals
+--------
+- None
+
+Upgrading from a previous version
+---------------------------------
+
+
+Using PIP
+~~~~~~~~~
+
+Type in the console::
+
+    $ pip install mayan-edms==3.0.1
+
+the requirements will also be updated automatically.
+
+
+Using Git
+~~~~~~~~~
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+~~~~~~~~~~~~
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+=============================
+
+* None
+
+Bugs fixed or issues closed
+===========================
+
+* `GitLab issue #486 <https://gitlab.com/mayan-edms/mayan-edms/issues/486>`_ Docker Verison 3.0 not working
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -22,6 +22,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.1
    3.0.1
    3.0
 

mayan/apps/checkouts/managers.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import, unicode_literals
 
 import logging
 
+from django.apps import apps
 from django.db import models
 from django.utils.timezone import now
 
@@ -18,37 +19,13 @@ logger = logging.getLogger(__name__)
 
 
 class DocumentCheckoutManager(models.Manager):
-    def checkout_document(self, document, expiration_datetime, user, block_new_version=True):
-        return self.create(
-            document=document, expiration_datetime=expiration_datetime,
-            user=user, block_new_version=block_new_version
-        )
-
-    def checked_out_documents(self):
-        return Document.objects.filter(
-            pk__in=self.model.objects.all().values_list(
-                'document__pk', flat=True
-            )
-        )
-
-    def expired_check_outs(self):
-        expired_list = Document.objects.filter(
-            pk__in=self.model.objects.filter(
-                expiration_datetime__lte=now()
-            ).values_list('document__pk', flat=True)
-        )
-        logger.debug('expired_list: %s', expired_list)
-        return expired_list
-
-    def check_in_expired_check_outs(self):
-        for document in self.expired_check_outs():
-            document.check_in()
-
-    def is_document_checked_out(self, document):
-        if self.model.objects.filter(document=document):
+    def are_document_new_versions_allowed(self, document, user=None):
+        try:
+            checkout_info = self.document_checkout_info(document)
+        except DocumentNotCheckedOut:
             return True
         else:
-            return False
+            return not checkout_info.block_new_version
 
     def check_in_document(self, document, user=None):
         try:
@@ -68,6 +45,23 @@ class DocumentCheckoutManager(models.Manager):
 
             document_checkout.delete()
 
+    def check_in_expired_check_outs(self):
+        for document in self.expired_check_outs():
+            document.check_in()
+
+    def checkout_document(self, document, expiration_datetime, user, block_new_version=True):
+        return self.create(
+            document=document, expiration_datetime=expiration_datetime,
+            user=user, block_new_version=block_new_version
+        )
+
+    def checked_out_documents(self):
+        return Document.objects.filter(
+            pk__in=self.model.objects.all().values_list(
+                'document__pk', flat=True
+            )
+        )
+
     def document_checkout_info(self, document):
         try:
             return self.model.objects.get(document=document)
@@ -80,21 +74,50 @@ class DocumentCheckoutManager(models.Manager):
         else:
             return STATE_CHECKED_IN
 
-    def are_document_new_versions_allowed(self, document, user=None):
+    def expired_check_outs(self):
+        expired_list = Document.objects.filter(
+            pk__in=self.model.objects.filter(
+                expiration_datetime__lte=now()
+            ).values_list('document__pk', flat=True)
+        )
+        logger.debug('expired_list: %s', expired_list)
+        return expired_list
+
+    def get_by_natural_key(self, document_natural_key):
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
         try:
-            checkout_info = self.document_checkout_info(document)
-        except DocumentNotCheckedOut:
+            document = Document.objects.get_by_natural_key(document_natural_key)
+        except Document.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(document__pk=document.pk)
+
+    def is_document_checked_out(self, document):
+        if self.model.objects.filter(document=document):
             return True
         else:
-            return not checkout_info.block_new_version
+            return False
 
 
 class NewVersionBlockManager(models.Manager):
     def block(self, document):
         self.get_or_create(document=document)
 
-    def unblock(self, document):
-        self.filter(document=document).delete()
-
     def is_blocked(self, document):
         return self.filter(document=document).exists()
+
+    def get_by_natural_key(self, document_natural_key):
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+        try:
+            document = Document.objects.get_by_natural_key(document_natural_key)
+        except Document.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(document__pk=document.pk)
+
+    def unblock(self, document):
+        self.filter(document=document).delete()

mayan/apps/checkouts/models.py

@@ -72,6 +72,10 @@ class DocumentCheckout(models.Model):
     def get_absolute_url(self):
         return reverse('checkout:checkout_info', args=(self.document.pk,))
 
+    def natural_key(self):
+        return self.document.natural_key()
+    natural_key.dependencies = ['documents.Document']
+
     def save(self, *args, **kwargs):
         # TODO: enclose in transaction
         new_checkout = not self.pk
@@ -104,3 +108,7 @@ class NewVersionBlock(models.Model):
     class Meta:
         verbose_name = _('New version block')
         verbose_name_plural = _('New version blocks')
+
+    def natural_key(self):
+        return self.document.natural_key()
+    natural_key.dependencies = ['documents.Document']

mayan/apps/common/javascript.py

@@ -8,7 +8,7 @@ import shutil
 import tarfile
 
 from furl import furl
-from pathlib import Path
+from pathlib2 import Path
 import requests
 from semver import max_satisfying
 

mayan/apps/common/management/commands/convertdb.py

@@ -0,0 +1,86 @@
+from __future__ import unicode_literals
+
+import errno
+import os
+
+from pathlib2 import Path
+
+from django.conf import settings
+from django.core import management
+from django.core.management.base import CommandError
+from django.utils.encoding import force_text
+from django.utils.translation import ugettext_lazy as _
+
+from common.utils import fs_cleanup
+from documents.models import DocumentType
+
+CONVERTDB_FOLDER = 'convertdb'
+CONVERTDB_OUTPUT_FILENAME = 'migrate.json'
+
+
+class Command(management.BaseCommand):
+    help = 'Convert from a database backend to another one.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--from', action='store', default='default', dest='from',
+            help=_(
+                'The database from which data will be exported. If omitted '
+                'the database named "default" will be used.'
+            ),
+        )
+        parser.add_argument(
+            '--to', action='store', default='default', dest='to',
+            help=_(
+                'The database to which data will be imported. If omitted '
+                'the database named "default" will be used.'
+            ),
+        )
+        parser.add_argument(
+            '--force', action='store_true', dest='force',
+            help=_(
+                'Force the conversion of the database even if the receving '
+                'database is not empty.'
+            ),
+        )
+
+    def handle(self, *args, **options):
+        # Create the media/convertdb folder
+        convertdb_folder_path = force_text(
+            Path(
+                settings.MEDIA_ROOT, CONVERTDB_FOLDER
+            )
+        )
+
+        try:
+            os.makedirs(convertdb_folder_path)
+        except OSError as exception:
+            if exception.errno == errno.EEXIST:
+                pass
+
+        convertdb_file_path = force_text(
+            Path(
+                convertdb_folder_path, CONVERTDB_OUTPUT_FILENAME
+            )
+        )
+
+        management.call_command(
+            'dumpdata', all=True, database=options['from'],
+            natural_primary=True, natural_foreign=True,
+            output=convertdb_file_path, interactive=False,
+            format='json'
+        )
+
+        if DocumentType.objects.using('default').count() and not options['force']:
+            fs_cleanup(convertdb_file_path)
+            raise CommandError(
+                'There is existing data in the database that will be '
+                'used for the import. If you proceed with the conversion '
+                'you might lose data. Please check you settings.'
+            )
+
+        management.call_command(
+            'loaddata', convertdb_file_path, database=options['to'],
+            interactive=False
+        )
+        fs_cleanup(convertdb_file_path)

mayan/apps/common/managers.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
+from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.fields import GenericRelation
 from django.db import models
 
@@ -11,3 +12,14 @@ class ErrorLogEntryManager(models.Manager):
             app_label='common', model_name='ErrorLogEntry'
         )
         model.add_to_class('error_logs', GenericRelation(ErrorLogEntry))
+
+
+class UserLocaleProfileManager(models.Manager):
+    def get_by_natural_key(self, user_natural_key):
+        User = get_user_model()
+        try:
+            user = User.objects.get_by_natural_key(user_natural_key)
+        except User.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(user__pk=user.pk)

mayan/apps/common/models.py

@@ -11,7 +11,7 @@ from django.db import models
 from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
-from .managers import ErrorLogEntryManager
+from .managers import ErrorLogEntryManager, UserLocaleProfileManager
 from .storages import storage_sharedupload
 
 
@@ -88,9 +88,16 @@ class UserLocaleProfile(models.Model):
         choices=settings.LANGUAGES, max_length=8, verbose_name=_('Language')
     )
 
+    objects = UserLocaleProfileManager()
+
     class Meta:
         verbose_name = _('User locale profile')
         verbose_name_plural = _('User locale profiles')
 
     def __str__(self):
         return force_text(self.user)
+
+    def natural_key(self):
+        return self.user.natural_key()
+    natural_key.dependencies = [settings.AUTH_USER_MODEL]
+

mayan/apps/document_indexing/managers.py

@@ -9,8 +9,8 @@ class DocumentIndexInstanceNodeManager(models.Manager):
 
 
 class IndexManager(models.Manager):
-    def get_by_natural_key(self, name):
-        return self.get(name=name)
+    def get_by_natural_key(self, slug):
+        return self.get(slug=slug)
 
     def index_document(self, document):
         for index in self.filter(enabled=True, document_types=document.document_type):

mayan/apps/document_indexing/models.py

@@ -83,6 +83,9 @@ class Index(models.Model):
     def instance_root(self):
         return self.template_root.index_instance_nodes.get()
 
+    def natural_key(self):
+        return (self.slug,)
+
     def rebuild(self):
         """
         Delete and reconstruct the index by deleting of all its instance nodes

mayan/apps/document_parsing/managers.py

@@ -4,6 +4,7 @@ import logging
 import sys
 import traceback
 
+from django.apps import apps
 from django.conf import settings
 from django.db import models
 
@@ -48,3 +49,16 @@ class DocumentPageContentManager(models.Manager):
                 action_object=document_version.document,
                 target=document_version
             )
+
+
+class DocumentTypeSettingsManager(models.Manager):
+    def get_by_natural_key(self, document_type_natural_key):
+        DocumentType = apps.get_model(
+            app_label='documents', model_name='DocumentType'
+        )
+        try:
+            document_type = DocumentType.objects.get_by_natural_key(document_type_natural_key)
+        except DocumentType.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(document_type__pk=document_type.pk)

mayan/apps/document_parsing/models.py

@@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from documents.models import DocumentPage, DocumentType, DocumentVersion
 
-from .managers import DocumentPageContentManager
+from .managers import DocumentPageContentManager, DocumentTypeSettingsManager
 
 
 @python_2_unicode_compatible
@@ -37,6 +37,12 @@ class DocumentTypeSettings(models.Model):
         verbose_name=_('Automatically queue newly created documents for parsing.')
     )
 
+    objects = DocumentTypeSettingsManager()
+
+    def natural_key(self):
+        return self.document_type.natural_key()
+    natural_key.dependencies = ['documents.DocumentType']
+
     class Meta:
         verbose_name = _('Document type settings')
         verbose_name_plural = _('Document types settings')

mayan/apps/documents/managers.py

@@ -4,8 +4,10 @@ from datetime import timedelta
 import logging
 
 from django.apps import apps
+from django.contrib.auth import get_user_model
 from django.db import models
 from django.db.models import F, Max
+from django.utils.encoding import force_text
 from django.utils.timezone import now
 
 from .literals import STUB_EXPIRATION_INTERVAL
@@ -20,7 +22,7 @@ class DocumentManager(models.Manager):
             stale_stub_document.delete(trash=False)
 
     def get_by_natural_key(self, uuid):
-        return self.get(uuid=uuid)
+        return self.model.passthrough.get(uuid=force_text(uuid))
 
     def get_queryset(self):
         return TrashCanQuerySet(
@@ -32,6 +34,32 @@ class DocumentManager(models.Manager):
             document.invalidate_cache()
 
 
+class DocumentPageManager(models.Manager):
+    def get_by_natural_key(self, page_number, document_version_natural_key):
+        DocumentVersion = apps.get_model(
+            app_label='documents', model_name='DocumentVersion'
+        )
+        try:
+            document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key)
+        except DocumentVersion.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(document_version__pk=document_version.pk, page_number=page_number)
+
+
+class DocumentVersionManager(models.Manager):
+    def get_by_natural_key(self, checksum, document_natural_key):
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+        try:
+            document = Document.objects.get_by_natural_key(*document_natural_key)
+        except Document.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(document__pk=document.pk, checksum=checksum)
+
+
 class DocumentTypeManager(models.Manager):
     def check_delete_periods(self):
         logger.info('Executing')
@@ -175,6 +203,26 @@ class RecentDocumentManager(models.Manager):
             self.filter(pk__in=list(recent_to_delete)).delete()
         return new_recent
 
+    def get_by_natural_key(self, datetime_accessed, document_natural_key, user_natural_key):
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+        User = get_user_model()
+        try:
+            document = Document.objects.get_by_natural_key(*document_natural_key)
+        except Document.DoesNotExist:
+            raise self.model.DoesNotExist
+        else:
+            try:
+                user = User.objects.get_by_natural_key(*user_natural_key)
+            except User.DoesNotExist:
+                raise self.model.DoesNotExist
+
+        return self.get(
+            document__pk=document.pk, user__pk=user.pk,
+            datetime_accessed=datetime_accessed
+        )
+
     def get_for_user(self, user):
         document_model = apps.get_model('documents', 'document')
 

mayan/apps/documents/models.py

@@ -33,8 +33,9 @@ from .events import (
 )
 from .literals import DEFAULT_DELETE_PERIOD, DEFAULT_DELETE_TIME_UNIT
 from .managers import (
-    DocumentManager, DocumentTypeManager, DuplicatedDocumentManager,
-    PassthroughManager, RecentDocumentManager, TrashCanManager
+    DocumentManager, DocumentPageManager, DocumentVersionManager,
+    DocumentTypeManager, DuplicatedDocumentManager, PassthroughManager,
+    RecentDocumentManager, TrashCanManager
 )
 from .permissions import permission_document_view
 from .settings import (
@@ -415,6 +416,8 @@ class DocumentVersion(models.Model):
         verbose_name = _('Document version')
         verbose_name_plural = _('Document version')
 
+    objects = DocumentVersionManager()
+
     def __str__(self):
         return self.get_rendered_string()
 
@@ -505,6 +508,10 @@ class DocumentVersion(models.Model):
             context=Context({'instance': self})
         )
 
+    def natural_key(self):
+        return (self.checksum, self.document.natural_key())
+    natural_key.dependencies = ['documents.Document']
+
     def invalidate_cache(self):
         storage_documentimagecache.delete(self.cache_filename)
         for page in self.pages.all():
@@ -728,6 +735,8 @@ class DocumentPage(models.Model):
         verbose_name=_('Page number')
     )
 
+    objects = DocumentPageManager()
+
     class Meta:
         ordering = ('page_number',)
         verbose_name = _('Document page')
@@ -887,6 +896,10 @@ class DocumentPage(models.Model):
         for cached_image in self.cached_images.all():
             cached_image.delete()
 
+    def natural_key(self):
+        return (self.page_number, self.document_version.natural_key())
+    natural_key.dependencies = ['documents.DocumentVersion']
+
     @property
     def siblings(self):
         return DocumentPage.objects.filter(
@@ -955,7 +968,7 @@ class RecentDocument(models.Model):
         return force_text(self.document)
 
     def natural_key(self):
-        return self.document.natural_key() + self.user.natural_key()
+        return (self.datetime_accessed, self.document.natural_key(), self.user.natural_key())
     natural_key.dependencies = ['documents.Document', settings.AUTH_USER_MODEL]
 
 

mayan/apps/mailer/managers.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+from django.db import models
+
+
+class UserMailerManager(models.Manager):
+    def get_by_natural_key(self, label):
+        return self.get(label=label)

mayan/apps/mailer/models.py

@@ -11,6 +11,7 @@ from django.utils.html import strip_tags
 from django.utils.module_loading import import_string
 from django.utils.translation import ugettext_lazy as _
 
+from .managers import UserMailerManager
 from .utils import split_recipient_list
 
 logger = logging.getLogger(__name__)
@@ -51,6 +52,8 @@ class UserMailer(models.Model):
         blank=True, verbose_name=_('Backend data')
     )
 
+    objects = UserMailerManager()
+
     class Meta:
         ordering = ('label',)
         verbose_name = _('User mailer')
@@ -77,6 +80,9 @@ class UserMailer(models.Model):
     def loads(self):
         return json.loads(self.backend_data)
 
+    def natural_key(self):
+        return (self.label,)
+
     def save(self, *args, **kwargs):
         if self.default:
             UserMailer.objects.select_for_update().exclude(pk=self.pk).update(

mayan/apps/metadata/managers.py

@@ -24,6 +24,25 @@ class MetadataTypeManager(models.Manager):
 
 
 class DocumentTypeMetadataTypeManager(models.Manager):
+    def get_by_natural_key(self, document_natural_key, metadata_type_natural_key):
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+        MetadataType = apps.get_model(
+            app_label='metadata', model_name='MetadataType'
+        )
+        try:
+            document = Document.objects.get_by_natural_key(document_natural_key)
+        except Document.DoesNotExist:
+            raise self.model.DoesNotExist
+        else:
+            try:
+                metadata_type = MetadataType.objects.get_by_natural_key(metadata_type_natural_key)
+            except MetadataType.DoesNotExist:
+                raise self.model.DoesNotExist
+
+        return self.get(document__pk=document.pk, metadata_type__pk=metadata_type.pk)
+
     def get_metadata_types_for(self, document_type):
         DocumentType = apps.get_model(
             app_label='metadata', model_name='MetadataType'

mayan/apps/metadata/models.py

@@ -189,6 +189,10 @@ class DocumentMetadata(models.Model):
 
         return super(DocumentMetadata, self).delete(*args, **kwargs)
 
+    def natural_key(self):
+        return self.document.natural_key() + self.metadata_type.natural_key()
+    natural_key.dependencies = ['documents.Document', 'metadata.MetadataType']
+
     @property
     def is_required(self):
         return self.metadata_type.get_required_for(

mayan/apps/ocr/managers.py

@@ -90,3 +90,16 @@ class DocumentPageOCRContentManager(models.Manager):
             post_document_version_ocr.send(
                 sender=document_version.__class__, instance=document_version
             )
+
+
+class DocumentTypeSettingsManager(models.Manager):
+    def get_by_natural_key(self, document_type_natural_key):
+        DocumentType = apps.get_model(
+            app_label='documents', model_name='DocumentType'
+        )
+        try:
+            document_type = DocumentType.objects.get_by_natural_key(document_type_natural_key)
+        except DocumentType.DoesNotExist:
+            raise self.model.DoesNotExist
+
+        return self.get(document_type__pk=document_type.pk)

mayan/apps/ocr/models.py

@@ -6,7 +6,9 @@ from django.utils.translation import ugettext_lazy as _
 
 from documents.models import DocumentPage, DocumentType, DocumentVersion
 
-from .managers import DocumentPageOCRContentManager
+from .managers import (
+    DocumentPageOCRContentManager, DocumentTypeSettingsManager
+)
 
 
 class DocumentTypeSettings(models.Model):
@@ -22,10 +24,17 @@ class DocumentTypeSettings(models.Model):
         verbose_name=_('Automatically queue newly created documents for OCR.')
     )
 
+    objects = DocumentTypeSettingsManager()
+
     class Meta:
         verbose_name = _('Document type settings')
         verbose_name_plural = _('Document types settings')
 
+    def natural_key(self):
+        return self.document_type.natural_key()
+    natural_key.dependencies = ['documents.DocumentType']
+
+
 
 @python_2_unicode_compatible
 class DocumentPageOCRContent(models.Model):

removals.txt

@@ -1,5 +1,7 @@
 # Packages to be remove during upgrades
+django-celery
 django-filetransfers
 django-rest-swagger
+pathlib
 pytesseract
 pdfminer

requirements/base.txt

@@ -6,7 +6,8 @@ cssmin==0.2.0
 
 django-activity-stream==0.6.5
 django-autoadmin==1.1.1
-django-celery==3.2.1
+#django-celery==3.2.1 - Use fork below until patch https://github.com/celery/django-celery/pull/552 is accepted.
+https://github.com/mayan-edms/django-celery/zipball/master#egg=django-celery
 django-colorful==1.2
 django-compressor==2.2
 django-cors-headers==2.2.0
@@ -38,7 +39,7 @@ mock==2.0.0
 
 node-semver==0.3.0
 
-pathlib==1.0.1
+pathlib2==2.3.2
 pycountry==18.5.26
 PyPDF2==1.26.0
 pyocr==0.5.1