Generate new metadata when label is ambiguous

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

HISTORY.rst

@@ -1,7 +1,38 @@
-3.2.2 (2019-06-XX)
+Importer branch
+===============
+* Add a reusable task to upload documents.
+* Add MVP of the importer app.
+
+3.2.4 (2019-06-XX)
+==================
+* Support configurable GUnicorn timeouts. Defaults to
+  current value of 120 seconds.
+
+3.2.3 (2019-06-21)
+==================
+* Add support for disabling the random primary key
+  test mixin.
+* Add a reusable task to upload documents.
+* Add MVP of the importer app.
+* Fix mailing profile log columns mappings.
+  GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
+  for the report.
+* Fix the Django SMTP backend username field name.
+  GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
+  for the report and the research.
+* Increase the Django STMP username.
+  GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
+  for the report and the research.
+
+3.2.2 (2019-06-19)
 ==================
 * Fix document type change view. Closes GitLab issue #614
   Thanks to Christoph Roeder (@brightdroid) for the report.
+* Fix document parsing tool view typo. Closes GitLab issue #615.
+  Thanks to Tyler Page (@iamtpage) for the report.
+* Update the task_check_interval_source reference
+  GitLab issue #617. Thanks to Lukas Gill (@lukkigi) for
+  the report and debug information.
 
 3.2.1 (2019-06-14)
 ==================

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -22,6 +22,7 @@ export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
 
 export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
 export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
 export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
 export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
 

docker/rootfs/version

@@ -1 +1 @@
-3.2.1
+3.2.3

docs/releases/3.2.2.rst

@@ -1,7 +1,7 @@
 Version 3.2.2
 =============
 
-Released: June 17, 2019
+Released: June 19, 2019
 
 
 Changes
@@ -9,6 +9,11 @@ Changes
 
 - Fix document type change view. Closes GitLab issue #614.
   Thanks to Christoph Roeder (@brightdroid) for the report.
+- Fix document parsing tool view typo. Closes GitLab issue #615.
+  Thanks to Tyler Page (@iamtpage) for the report.
+- Update the task_check_interval_source reference
+  GitLab issue #617. Thanks to Lukas Gill (@lukkigi) for
+  the report and debug information.
 
 Removals
 --------
@@ -28,7 +33,7 @@ Remove deprecated requirements::
 
 Type in the console::
 
-    $ pip install mayan-edms==3.2.1
+    $ pip install mayan-edms==3.2.2
 
 the requirements will also be updated automatically.
 
@@ -98,5 +103,7 @@ Bugs fixed or issues closed
 ---------------------------
 
 - :gitlab-issue:`614` change type exception
+- :gitlab-issue:`615` TypeError: success() got an unexpected keyword argument 'requrest'
+- :gitlab-issue:`617` Watcher Task not running
 
 .. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.3.rst

@@ -0,0 +1,113 @@
+Version 3.2.3
+=============
+
+Released: June 21, 2019
+
+
+Changes
+-------
+
+- Add support for disabling the random primary key
+  test mixin.
+- Fix mailing profile log columns mappings.
+  GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
+  for the report.
+- Fix the Django SMTP backend username field name.
+  GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
+  for the report and the research.
+- Increase the Django STMP username.
+  GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
+  for the report and the research.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.1
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`619` poplib.POP3_SSL and poplib.POP3 initialized with wrong kwarg
+- :gitlab-issue:`625` mayan.apps.mailer.mailers.DjangoSMTP uses "user", but django.core.mail.backends.smtp.EmailBackend expects "username"
+- :gitlab-issue:`626` Mailing profile error log is empty, despite errors
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.4.rst

@@ -0,0 +1,103 @@
+Version 3.2.4
+=============
+
+Released: June XX, 2019
+
+
+Changes
+-------
+
+- Support configurable GUnicorn timeouts. Defaults to
+  current value of 120 seconds.
+
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.3
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`628` mailbox.user in POP3Email gets passed keyword argument, but only accepts "user" or positional argument
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -20,6 +20,8 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.2.4
+   3.2.3
    3.2.2
    3.2.1
    3.2

mayan/__init__.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '3.2.1'
-__build__ = 0x030201
-__build_string__ = 'v3.2.1_Fri Jun 14 03:01:40 2019 -0400'
+__version__ = '3.2.3'
+__build__ = 0x030203
+__build_string__ = 'v3.2.3_Fri Jun 21 00:01:37 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'

mayan/apps/common/tests/mixins.py

@@ -144,6 +144,7 @@ class RandomPrimaryKeyModelMonkeyPatchMixin(object):
     random_primary_key_random_floor = 100
     random_primary_key_random_ceiling = 10000
     random_primary_key_maximum_attempts = 100
+    random_primary_key_enable = True
 
     @staticmethod
     def get_unique_primary_key(model):
@@ -170,47 +171,49 @@ class RandomPrimaryKeyModelMonkeyPatchMixin(object):
         return primary_key
 
     def setUp(self):
-        self.method_save_original = models.Model.save
+        if self.random_primary_key_enable:
+            self.method_save_original = models.Model.save
 
-        def method_save_new(instance, *args, **kwargs):
-            if instance.pk:
-                return self.method_save_original(instance, *args, **kwargs)
-            else:
-                # Set meta.auto_created to True to have the original save_base
-                # not send the pre_save signal which would normally send
-                # the instance without a primary key. Since we assign a random
-                # primary key any pre_save signal handler that relies on an
-                # empty primary key will fail.
-                # The meta.auto_created and manual pre_save sending emulates
-                # the original behavior. Since meta.auto_created also disables
-                # the post_save signal we must also send it ourselves.
-                # This hack work with Django 1.11 .save_base() but can break
-                # in future versions if that method is updated.
-                pre_save.send(
-                    sender=instance.__class__, instance=instance, raw=False,
-                    update_fields=None,
-                )
-                instance._meta.auto_created = True
-                instance.pk = RandomPrimaryKeyModelMonkeyPatchMixin.get_unique_primary_key(
-                    model=instance._meta.model
-                )
-                instance.id = instance.pk
+            def method_save_new(instance, *args, **kwargs):
+                if instance.pk:
+                    return self.method_save_original(instance, *args, **kwargs)
+                else:
+                    # Set meta.auto_created to True to have the original save_base
+                    # not send the pre_save signal which would normally send
+                    # the instance without a primary key. Since we assign a random
+                    # primary key any pre_save signal handler that relies on an
+                    # empty primary key will fail.
+                    # The meta.auto_created and manual pre_save sending emulates
+                    # the original behavior. Since meta.auto_created also disables
+                    # the post_save signal we must also send it ourselves.
+                    # This hack work with Django 1.11 .save_base() but can break
+                    # in future versions if that method is updated.
+                    pre_save.send(
+                        sender=instance.__class__, instance=instance, raw=False,
+                        update_fields=None,
+                    )
+                    instance._meta.auto_created = True
+                    instance.pk = RandomPrimaryKeyModelMonkeyPatchMixin.get_unique_primary_key(
+                        model=instance._meta.model
+                    )
+                    instance.id = instance.pk
 
-                result = instance.save_base(force_insert=True)
-                instance._meta.auto_created = False
+                    result = instance.save_base(force_insert=True)
+                    instance._meta.auto_created = False
 
-                post_save.send(
-                    sender=instance.__class__, instance=instance, created=True,
-                    update_fields=None, raw=False
-                )
+                    post_save.send(
+                        sender=instance.__class__, instance=instance, created=True,
+                        update_fields=None, raw=False
+                    )
 
-                return result
+                    return result
 
-        setattr(models.Model, 'save', method_save_new)
+            setattr(models.Model, 'save', method_save_new)
         super(RandomPrimaryKeyModelMonkeyPatchMixin, self).setUp()
 
     def tearDown(self):
-        models.Model.save = self.method_save_original
+        if self.random_primary_key_enable:
+            models.Model.save = self.method_save_original
         super(RandomPrimaryKeyModelMonkeyPatchMixin, self).tearDown()
 
 

mayan/apps/document_parsing/tests/test_views.py

@@ -7,7 +7,8 @@ from mayan.apps.documents.tests import (
 )
 
 from ..permissions import (
-    permission_content_view, permission_document_type_parsing_setup
+    permission_content_view, permission_document_type_parsing_setup,
+    permission_parse_document
 )
 from ..utils import get_document_content
 
@@ -106,3 +107,43 @@ class DocumentContentViewsTestCase(GenericDocumentViewTestCase):
 
         response = self._request_test_document_type_parsing_settings()
         self.assertEqual(response.status_code, 200)
+
+
+class DocumentContentToolsViewsTestCase(GenericDocumentViewTestCase):
+    _skip_file_descriptor_test = True
+
+    # Ensure we use a PDF file
+    test_document_filename = TEST_HYBRID_DOCUMENT
+
+    def _request_document_parsing_tool_view(self):
+        return self.post(
+            viewname='document_parsing:document_type_submit', data={
+                'document_type': self.test_document_type.pk
+            }
+        )
+
+    def _get_document_content(self):
+        return ''.join(list(get_document_content(document=self.test_document)))
+
+    def test_document_parsing_tool_view_no_permission(self):
+
+        response = self._request_document_parsing_tool_view()
+        self.assertEqual(response.status_code, 200)
+        self.assertNotContains(
+            response=response, status_code=200,
+            text=self.test_document_type.label
+        )
+
+        self.assertNotEqual(
+            self._get_document_content(), TEST_DOCUMENT_CONTENT
+        )
+
+    def test_document_parsing_tool_view_with_permission(self):
+        self.grant_permission(permission=permission_parse_document)
+
+        response = self._request_document_parsing_tool_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertEqual(
+            self._get_document_content(), TEST_DOCUMENT_CONTENT
+        )

mayan/apps/document_parsing/views.py

@@ -185,7 +185,7 @@ class DocumentTypeSubmitView(FormView):
                 '%(count)d documents added to the parsing queue.'
             ) % {
                 'count': count,
-            }, requrest=self.request
+            }, request=self.request
         )
 
         return HttpResponseRedirect(redirect_to=self.get_success_url())

mayan/apps/documents/literals.py

@@ -32,6 +32,7 @@ DEFAULT_DOCUMENT_TYPE_LABEL = _('Default')
 DOCUMENT_IMAGE_TASK_TIMEOUT = 120
 STUB_EXPIRATION_INTERVAL = 60 * 60 * 24  # 24 hours
 UPDATE_PAGE_COUNT_RETRY_DELAY = 10
+UPLOAD_NEW_DOCUMENT_RETRY_DELAY = 10
 UPLOAD_NEW_VERSION_RETRY_DELAY = 10
 
 PAGE_RANGE_ALL = 'all'

mayan/apps/documents/queues.py

@@ -82,3 +82,7 @@ queue_uploads.add_task_type(
     dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for',
     label=_('Scan document duplicates')
 )
+queue_uploads.add_task_type(
+    dotted_path='mayan.apps.documents.tasks.task_upload_new_document',
+    label=_('Upload new document')
+)

mayan/apps/documents/tasks.py

@@ -9,7 +9,8 @@ from django.db import OperationalError
 from mayan.celery import app
 
 from .literals import (
-    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY
+    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY,
+    UPLOAD_NEW_VERSION_RETRY_DELAY
 )
 
 logger = logging.getLogger(__name__)
@@ -127,6 +128,60 @@ def task_update_page_count(self, version_id):
         raise self.retry(exc=exception)
 
 
+@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
+def task_upload_new_document(self, document_type_id, shared_uploaded_file_id):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+
+    SharedUploadedFile = apps.get_model(
+        app_label='common', model_name='SharedUploadedFile'
+    )
+
+    try:
+        document_type = DocumentType.objects.get(pk=document_type_id)
+        shared_file = SharedUploadedFile.objects.get(
+            pk=shared_uploaded_file_id
+        )
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to retrieve shared data for '
+            'new document of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+
+    try:
+        with shared_file.open() as file_object:
+            document_type.new_document(file_object=file_object)
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to create new document '
+            'of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+    except Exception as exception:
+        # This except and else block emulate a finally:
+        logger.error(
+            'Unexpected error during attempt to create new document '
+            'of type: %s; %s', document_type, exception
+        )
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+    else:
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+
+
 @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True)
 def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None):
     SharedUploadedFile = apps.get_model(

mayan/apps/importer/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'mayan.apps.importer.apps.ImporterApp'

mayan/apps/importer/apps.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.common.apps import MayanAppConfig
+
+
+class ImporterApp(MayanAppConfig):
+    app_namespace = 'importer'
+    app_url = 'importer'
+    has_rest_api = False
+    has_tests = True
+    name = 'mayan.apps.importer'
+    verbose_name = _('Importer')
+
+    def ready(self):
+        super(ImporterApp, self).ready()

mayan/apps/importer/management/commands/import.py

@@ -0,0 +1,150 @@
+from __future__ import unicode_literals
+
+import csv
+import time
+
+from django.apps import apps
+from django.core import management
+from django.core.files import File
+
+from ...tasks import task_upload_new_document
+
+
+class Command(management.BaseCommand):
+    help = 'Import documents from a CSV file.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--document_type_column',
+            action='store', dest='document_type_column', default=0,
+            help='Column that contains the document type labels. Column '
+            'numbers start at 0.',
+            type=int
+        )
+        parser.add_argument(
+            '--document_path_column',
+            action='store', dest='document_path_column', default=1,
+            help='Column that contains the path to the document files. Column '
+            'numbers start at 0.',
+            type=int
+        )
+        parser.add_argument(
+            '--ignore_errors',
+            action='store_true', dest='ignore_errors', default=False,
+            help='Don\'t stop the import process on common errors like '
+            'incorrect file paths.',
+        )
+        parser.add_argument(
+            '--ignore_rows',
+            action='store', dest='ignore_rows', default='',
+            help='Ignore a set of rows. Row numbers must be separated by commas.'
+        )
+        parser.add_argument(
+            '--metadata_pairs_column',
+            action='store', dest='metadata_pairs_column',
+            help='Column that contains metadata name and values for the '
+            'documents. Use the form: <label column>:<value column>. Example: '
+            '2:5. Separate multiple pairs with commas. Example: 2:5,7:10',
+        )
+        parser.add_argument('filelist', nargs='?', help='File list')
+
+    def handle(self, *args, **options):
+        time_start = time.time()
+        time_last_display = time_start
+        document_types = {}
+        uploaded_count = 0
+        row_count = 0
+        rows_to_ignore = []
+        for entry in options['ignore_rows'].split(','):
+            if entry:
+                rows_to_ignore.append(int(entry))
+
+        DocumentType = apps.get_model(
+            app_label='documents', model_name='DocumentType'
+        )
+        SharedUploadedFile = apps.get_model(
+            app_label='common', model_name='SharedUploadedFile'
+        )
+
+        if not options['filelist']:
+            self.stderr.write('Must specify a CSV file path.')
+            exit(1)
+        else:
+            with open(options['filelist']) as csv_datafile:
+                csv_reader = csv.reader(csv_datafile)
+                for row in csv_reader:
+                    # Increase row count here even though start index is 0
+                    # purpose is to avoid losing row number increments on
+                    # exceptions
+                    row_count = row_count + 1
+                    if row_count - 1 not in rows_to_ignore:
+                        try:
+                            with open(row[options['document_path_column']]) as file_object:
+                                document_type_label = row[options['document_type_column']]
+
+                                if document_type_label not in document_types:
+                                    self.stdout.write(
+                                        'New document type: {}. Creating and caching.'.format(
+                                            document_type_label
+                                        )
+                                    )
+                                    document_type, created = DocumentType.objects.get_or_create(
+                                        label=document_type_label
+                                    )
+                                    document_types[document_type_label] = document_type
+                                else:
+                                    document_type = document_types[document_type_label]
+
+                                shared_uploaded_file = SharedUploadedFile.objects.create(
+                                    file=File(file_object)
+                                )
+
+                                extra_data = {}
+                                if options['metadata_pairs_column']:
+                                    extra_data['metadata_pairs'] = []
+
+                                    for pair in options['metadata_pairs_column'].split(','):
+                                        name, value = pair.split(':')
+                                        extra_data['metadata_pairs'].append(
+                                            {
+                                                'name': row[int(name)],
+                                                'value': row[int(value)]
+                                            }
+                                        )
+
+                                task_upload_new_document.apply_async(
+                                    kwargs=dict(
+                                        document_type_id=document_type.pk,
+                                        shared_uploaded_file_id=shared_uploaded_file.pk,
+                                        extra_data=extra_data
+                                    )
+                                )
+
+                                uploaded_count = uploaded_count + 1
+
+                                if (time.time() - time_last_display) > 1:
+                                    time_last_display = time.time()
+                                    self.stdout.write(
+                                        'Time: {}s, Files copied and queued: {}, files processed per second: {}'.format(
+                                            int(time.time() - time_start),
+                                            uploaded_count,
+                                            uploaded_count / (time.time() - time_start)
+                                        )
+                                    )
+
+                        except (IOError, OSError) as exception:
+                            if not options['ignore_errors']:
+                                raise
+                            else:
+                                self.stderr.write(
+                                    'Error processing row: {}; {}.'.format(
+                                        row_count - 1, exception
+                                    )
+                                )
+
+            self.stdout.write(
+                'Total files copied and queues: {}'.format(uploaded_count)
+            )
+            self.stdout.write(
+                'Total time: {}'.format(time.time() - time_start)
+            )

mayan/apps/importer/queues.py

@@ -0,0 +1,10 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.documents.queues import queue_uploads
+
+queue_uploads.add_task_type(
+    dotted_path='mayan.apps.importer.tasks.task_upload_new_document',
+    label=_('Import new document')
+)

mayan/apps/importer/tasks.py

@@ -0,0 +1,92 @@
+from __future__ import unicode_literals
+
+import logging
+
+from django.apps import apps
+from django.db import OperationalError
+from django.utils.text import slugify
+
+from mayan.celery import app
+
+from mayan.apps.documents.literals import UPLOAD_NEW_DOCUMENT_RETRY_DELAY
+
+logger = logging.getLogger(__name__)
+
+
+@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
+def task_upload_new_document(self, document_type_id, shared_uploaded_file_id, extra_data=None):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+
+    MetadataType = apps.get_model(
+        app_label='metadata', model_name='MetadataType'
+    )
+
+    SharedUploadedFile = apps.get_model(
+        app_label='common', model_name='SharedUploadedFile'
+    )
+
+    try:
+        document_type = DocumentType.objects.get(pk=document_type_id)
+        shared_file = SharedUploadedFile.objects.get(
+            pk=shared_uploaded_file_id
+        )
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to retrieve shared data for '
+            'new document of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+
+    try:
+        with shared_file.open() as file_object:
+            new_document = document_type.new_document(file_object=file_object)
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to create new document '
+            'of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+    except Exception as exception:
+        # This except and else block emulate a finally:
+        logger.error(
+            'Unexpected error during attempt to create new document '
+            'of type: %s; %s', document_type, exception
+        )
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+    else:
+        if extra_data:
+            for pair in extra_data.get('metadata_pairs', []):
+                name = slugify(pair['name']).replace('-', '_')
+                logger.debug(
+                    'Metadata pair (label, name, value): %s, %s, %s',
+                    pair['name'], name, pair['value']
+                )
+
+                metadata_type, created = MetadataType.objects.get_or_create(
+                    label=pair['name'], name=name
+                )
+                if not new_document.document_type.metadata.filter(metadata_type=metadata_type).exists():
+                    logger.debug('Metadata type created')
+                    new_document.document_type.metadata.create(
+                        metadata_type=metadata_type, required=False
+                    )
+
+                new_document.metadata.create(
+                    metadata_type=metadata_type, value=pair['value']
+                )
+
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )

mayan/apps/importer/tests/test_management_commands.py

@@ -0,0 +1,94 @@
+from __future__ import unicode_literals
+
+import csv
+
+from django.core import management
+from django.utils.encoding import force_bytes
+
+from mayan.apps.documents.models import DocumentType, Document
+from mayan.apps.documents.tests import GenericDocumentTestCase
+from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH
+from mayan.apps.storage.utils import fs_cleanup, mkstemp
+
+
+class ImportManagementCommandTestCase(GenericDocumentTestCase):
+    auto_upload_document = False
+    random_primary_key_enable = False
+    test_import_count = 1
+
+    def setUp(self):
+        super(ImportManagementCommandTestCase, self).setUp()
+        self._create_test_csv_file()
+
+    def tearDown(self):
+        self._destroy_test_csv_file()
+        super(ImportManagementCommandTestCase, self).tearDown()
+
+    def _create_test_csv_file(self):
+        self.test_csv_file_descriptor, self.test_csv_path = mkstemp()
+
+        print('Test CSV file: {}'.format(self.test_csv_path))
+
+        with open(self.test_csv_path, mode='wb') as csvfile:
+            filewriter = csv.writer(
+                csvfile, delimiter=force_bytes(','), quotechar=force_bytes('"'),
+                quoting=csv.QUOTE_MINIMAL
+            )
+            print(
+                'Generating test CSV for {} documents'.format(
+                    self.test_import_count
+                )
+            )
+            for times in range(self.test_import_count):
+                filewriter.writerow(
+                    [
+                        self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH,
+                        'column 2', 'column 3', 'column 4', 'column 5',
+                        'column 6', 'column 7', 'column 8', 'column 9',
+                        'column 10', 'column 11',
+                    ]
+                )
+
+    def _destroy_test_csv_file(self):
+        fs_cleanup(
+            filename=self.test_csv_path,
+            file_descriptor=self.test_csv_file_descriptor
+        )
+
+    def test_import_csv_read(self):
+        self.test_document_type.delete()
+        management.call_command('import', self.test_csv_path)
+
+        self.assertTrue(DocumentType.objects.count() > 0)
+        self.assertTrue(Document.objects.count() > 0)
+
+    def test_import_document_type_column_mapping(self):
+        self.test_document_type.delete()
+        management.call_command(
+            'import', self.test_csv_path, '--document_type_column', '2'
+        )
+
+        self.assertTrue(DocumentType.objects.first().label == 'column 2')
+        self.assertTrue(Document.objects.count() > 0)
+
+    def test_import_document_path_column_mapping(self):
+        self.test_document_type.delete()
+        with self.assertRaises(IOError):
+            management.call_command(
+                'import', self.test_csv_path, '--document_path_column', '2'
+            )
+
+    def test_import_metadata_column_mapping(self):
+        self.test_document_type.delete()
+        management.call_command(
+            'import', self.test_csv_path, '--metadata_pairs_column', '2:3,4:5',
+        )
+
+        self.assertTrue(DocumentType.objects.count() > 0)
+        self.assertTrue(Document.objects.count() > 0)
+        self.assertTrue(Document.objects.first().metadata.count() > 0)
+        self.assertEqual(
+            Document.objects.first().metadata.get(
+                metadata_type__name='column_2'
+            ).value, 'column 3'
+        )

mayan/apps/mailer/apps.py

@@ -51,6 +51,7 @@ class MailerApp(MayanAppConfig):
 
         LogEntry = self.get_model(model_name='LogEntry')
         UserMailer = self.get_model(model_name='UserMailer')
+        UserMailerLogEntry = self.get_model(model_name='UserMailerLogEntry')
 
         MailerBackend.initialize()
 
@@ -79,6 +80,13 @@ class MailerApp(MayanAppConfig):
         SourceColumn(
             source=UserMailer, attribute='backend_label'
         )
+        SourceColumn(
+            attribute='datetime', label=_('Date and time'),
+            source=UserMailerLogEntry
+        )
+        SourceColumn(
+            attribute='message', label=_('Message'), source=UserMailerLogEntry
+        )
 
         ModelPermission.register(
             model=Document, permissions=(

mayan/apps/mailer/forms.py

@@ -96,7 +96,7 @@ class UserMailerDynamicForm(DynamicModelForm):
         if self.instance.backend_data:
             backend_data = json.loads(self.instance.backend_data)
             for key in self.instance.get_backend().fields:
-                self.fields[key].initial = backend_data[key]
+                self.fields[key].initial = backend_data.get(key)
 
         return result
 

mayan/apps/mailer/mailers.py

@@ -12,11 +12,11 @@ class DjangoSMTP(MailerBackend):
     Backend that wraps Django's SMTP backend
     """
     class_fields = (
-        'host', 'port', 'use_tls', 'use_ssl', 'user', 'password'
+        'host', 'port', 'use_tls', 'use_ssl', 'username', 'password'
     )
     class_path = 'django.core.mail.backends.smtp.EmailBackend'
     field_order = (
-        'host', 'port', 'use_tls', 'use_ssl', 'user', 'password', 'from'
+        'host', 'port', 'use_tls', 'use_ssl', 'username', 'password', 'from'
     )
     fields = {
         'from': {
@@ -60,14 +60,14 @@ class DjangoSMTP(MailerBackend):
                 'that "Use TLS" and "Use SSL" are mutually exclusive, '
                 'so only set one of those settings to True.'
             ), 'required': False
-        }, 'user': {
+        }, 'username': {
             'label': _('Username'),
             'class': 'django.forms.CharField', 'default': '',
             'help_text': _(
                 'Username to use for the SMTP server. If empty, '
                 'authentication won\'t attempted.'
             ), 'kwargs': {
-                'max_length': 48
+                'max_length': 254
             }, 'required': False
         }, 'password': {
             'label': _('Password'),

mayan/apps/mailer/views.py

@@ -208,7 +208,7 @@ class UserMailerLogEntryListView(SingleObjectListView):
         return {
             'hide_object': True,
             'object': self.get_user_mailer(),
-            'title': _('%s error log') % self.get_user_mailer(),
+            'title': _('Error log for: %s') % self.get_user_mailer(),
         }
 
     def get_source_queryset(self):

mayan/apps/platform/classes.py

@@ -120,6 +120,10 @@ class PlatformTemplateSupervisord(PlatformTemplate):
             name='GUNICORN_WORKERS', default=2,
             environment_name='MAYAN_GUNICORN_WORKERS'
         ),
+        Variable(
+            name='GUNICORN_TIMEOUT', default=120,
+            environment_name='MAYAN_GUNICORN_TIMEOUT'
+        ),
         Variable(
             name='DATABASE_CONN_MAX_AGE', default=0,
             environment_name='MAYAN_DATABASE_CONN_MAX_AGE'

mayan/apps/platform/templates/platform/supervisord.tmpl

@@ -16,7 +16,7 @@ environment=
 [program:mayan-gunicorn]
 autorestart = true
 autostart = true
-command = {{ INSTALLATION_PATH }}/bin/gunicorn -w {{ GUNICORN_WORKERS }} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+command = {{ INSTALLATION_PATH }}/bin/gunicorn -w {{ GUNICORN_WORKERS }} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout {{ GUNICORN_TIMEOUT }}
 user = mayan
 
 {% for worker in workers %}

mayan/apps/platform/templates/platform/supervisord_docker.tmpl

@@ -1,7 +1,7 @@
 [program:mayan-gunicorn]
 autorestart = false
 autostart = true
-command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120
+command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT}
 redirect_stderr = true
 stderr_logfile = /dev/fd/2
 stderr_logfile_maxbytes = 0

mayan/apps/sources/models/base.py

@@ -234,7 +234,7 @@ class IntervalBaseModel(OutOfProcessSource):
         PeriodicTask.objects.create(
             name=self._get_periodic_task_name(),
             interval=interval_instance,
-            task='sources.tasks.task_check_interval_source',
+            task='mayan.apps.sources.tasks.task_check_interval_source',
             kwargs=json.dumps({'source_id': self.pk})
         )
 

mayan/apps/sources/models/email_sources.py

@@ -277,15 +277,15 @@ class POP3Email(EmailBaseModel):
         logger.debug('ssl: %s', self.ssl)
 
         if self.ssl:
-            mailbox = poplib.POP3_SSL(host=self.host, post=self.port)
+            mailbox = poplib.POP3_SSL(host=self.host, port=self.port)
         else:
             mailbox = poplib.POP3(
-                host=self.host, post=self.port, timeout=self.timeout
+                host=self.host, port=self.port, timeout=self.timeout
             )
 
         mailbox.getwelcome()
-        mailbox.user(username=self.username)
-        mailbox.pass_(password=self.password)
+        mailbox.user(self.username)
+        mailbox.pass_(self.password)
         messages_info = mailbox.list()
 
         logger.debug(msg='messages_info:')

mayan/apps/sources/tests/test_models.py

@@ -203,7 +203,10 @@ class POP3SourceTestCase(GenericDocumentTestCase):
         def list(self, which=None):
             return (None, ['1 test'])
 
-        def pass_(self, password):
+        def user(self, user):
+            return
+
+        def pass_(self, pswd):
             return
 
         def quit(self):
@@ -214,10 +217,7 @@ class POP3SourceTestCase(GenericDocumentTestCase):
                 1, [TEST_EMAIL_BASE64_FILENAME]
             )
 
-        def user(self, username):
-            return
-
-    @mock.patch('poplib.POP3_SSL')
+    @mock.patch('poplib.POP3_SSL', autospec=True)
     def test_download_document(self, mock_poplib):
         mock_poplib.return_value = POP3SourceTestCase.MockMailbox()
         self.source = POP3Email.objects.create(

mayan/settings/base.py

@@ -120,6 +120,7 @@ INSTALLED_APPS = (
     'mayan.apps.document_states',
     'mayan.apps.documents',
     'mayan.apps.file_metadata',
+    'mayan.apps.importer',
     'mayan.apps.linking',
     'mayan.apps.mailer',
     'mayan.apps.mayan_statistics',