Merge branch 'versions/minor' into features/workflow_email_action

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

.gitlab-ci.yml

@@ -63,6 +63,7 @@ job_docker_nightly:
   only:
     - nightly
     - staging
+    - /^clients\/.+$/
 
 job_documentation_build:
   stage: build_documentation
@@ -160,6 +161,7 @@ job_push_python:
     - releases/python
     - staging
     - nightly
+    - /^clients\/.+$/
 
 test-mysql:
   <<: *test_base

CHANGES_BC.rst

@@ -0,0 +1,14 @@
+- Use Select2 widget for the document type selection form.
+- Update source column matching to be additive and not exclusive.
+- Add two columns to show the number of documents per workflow and
+  workflow state.
+- Sort module.
+- Add link to sort individual indexes.
+- Support exclusions from source columns.
+- Improve source column exclusion. Improve for model subclasses in partial querysets.
+- Add sortable index instance label column.
+- Add rectangle drawing transformation.
+- Redactions app.
+- Remove duplicated trashed document preview.
+- Add label to trashed date and time document source column.
+- Tag created event fix.

HISTORY.rst

@@ -1,3 +1,8 @@
+Importer branch
+===============
+* Add a reusable task to upload documents.
+* Add MVP of the importer app.
+
 3.3 (2019-XX-XX)
 ================
 - Add support for icon shadows.
@@ -95,6 +100,8 @@
 ==================
 - Add support for disabling the random primary key
   test mixin.
+- Add a reusable task to upload documents.
+- Add MVP of the importer app.
 - Fix mailing profile log columns mappings.
   GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
   for the report.

docs/releases/3.2.3.rst

@@ -19,7 +19,6 @@ Changes
   GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
   for the report and the research.
 
-
 Removals
 --------
 

mayan/apps/converter/migrations/0014_auto_20190626_1904.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-26 19:04
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('converter', '0013_auto_20180823_2353'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='transformation',
+            name='name',
+            field=models.CharField(choices=[('crop', 'Crop: left, top, right, bottom'), ('draw_rectangle', 'Draw rectangle: left, top, right, bottom, fillcolor, outlinecolor, outlinewidth'), ('draw_rectangle_percent', 'Draw rectangle (percents coordinates): left, top, right, bottom, fillcolor, outlinecolor, outlinewidth'), ('flip', 'Flip'), ('gaussianblur', 'Gaussian blur: radius'), ('lineart', 'Line art'), ('mirror', 'Mirror'), ('resize', 'Resize: width, height'), ('rotate', 'Rotate: degrees, fillcolor'), ('rotate180', 'Rotate 180 degrees'), ('rotate270', 'Rotate 270 degrees'), ('rotate90', 'Rotate 90 degrees'), ('unsharpmask', 'Unsharp masking: radius, percent, threshold'), ('zoom', 'Zoom: percent')], max_length=128, verbose_name='Name'),
+        ),
+    ]

mayan/apps/converter/transformations.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import hashlib
 import logging
 
-from PIL import Image, ImageColor, ImageFilter
+from PIL import Image, ImageColor, ImageDraw, ImageFilter
 
 from django.utils.translation import string_concat, ugettext_lazy as _
 from django.utils.encoding import force_bytes
@@ -151,6 +151,214 @@ class TransformationCrop(BaseTransformation):
         return self.image.crop((left, top, right, bottom))
 
 
+class TransformationDrawRectangle(BaseTransformation):
+    arguments = (
+        'left', 'top', 'right', 'bottom', 'fillcolor', 'outlinecolor',
+        'outlinewidth'
+    )
+    label = _('Draw rectangle')
+    name = 'draw_rectangle'
+
+    def execute_on(self, *args, **kwargs):
+        super(TransformationDrawRectangle, self).execute_on(*args, **kwargs)
+
+        try:
+            left = int(self.left or '0')
+        except ValueError:
+            left = 0
+
+        try:
+            top = int(self.top or '0')
+        except ValueError:
+            top = 0
+
+        try:
+            right = int(self.right or '0')
+        except ValueError:
+            right = 0
+
+        try:
+            bottom = int(self.bottom or '0')
+        except ValueError:
+            bottom = 0
+
+        if left < 0:
+            left = 0
+
+        if left > self.image.size[0] - 1:
+            left = self.image.size[0] - 1
+
+        if top < 0:
+            top = 0
+
+        if top > self.image.size[1] - 1:
+            top = self.image.size[1] - 1
+
+        if right < 0:
+            right = 0
+
+        if right > self.image.size[0] - 1:
+            right = self.image.size[0] - 1
+
+        if bottom < 0:
+            bottom = 0
+
+        if bottom > self.image.size[1] - 1:
+            bottom = self.image.size[1] - 1
+
+        # Invert right value
+        # Pillow uses left, top, right, bottom to define a viewport
+        # of real coordinates
+        # We invert the right and bottom to define a viewport
+        # that can crop from the right and bottom borders without
+        # having to know the real dimensions of an image
+        right = self.image.size[0] - right
+        bottom = self.image.size[1] - bottom
+
+        if left > right:
+            left = right - 1
+
+        if top > bottom:
+            top = bottom - 1
+
+        logger.debug(
+            'left: %f, top: %f, right: %f, bottom: %f', left, top, right,
+            bottom
+        )
+
+        fillcolor_value = getattr(self, 'fillcolor', None)
+        if fillcolor_value:
+            fill_color = ImageColor.getrgb(fillcolor_value)
+        else:
+            fill_color = 0
+
+        outlinecolor_value = getattr(self, 'outlinecolor', None)
+        if outlinecolor_value:
+            outline_color = ImageColor.getrgb(outlinecolor_value)
+        else:
+            outline_color = None
+
+        outlinewidth_value = getattr(self, 'outlinewidth', None)
+        if outlinewidth_value:
+            outline_width = int(outlinewidth_value)
+        else:
+            outline_width = 0
+
+        draw = ImageDraw.Draw(self.image)
+        draw.rectangle(
+            (left, top, right, bottom), fill=fill_color, outline=outline_color,
+            width=outline_width
+        )
+
+        return self.image
+
+
+class TransformationDrawRectanglePercent(BaseTransformation):
+    arguments = (
+        'left', 'top', 'right', 'bottom', 'fillcolor', 'outlinecolor',
+        'outlinewidth'
+    )
+    label = _('Draw rectangle (percents coordinates)')
+    name = 'draw_rectangle_percent'
+
+    def execute_on(self, *args, **kwargs):
+        super(TransformationDrawRectanglePercent, self).execute_on(*args, **kwargs)
+
+        try:
+            left = float(self.left or '0')
+        except ValueError:
+            left = 0
+
+        try:
+            top = float(self.top or '0')
+        except ValueError:
+            top = 0
+
+        try:
+            right = float(self.right or '0')
+        except ValueError:
+            right = 0
+
+        try:
+            bottom = float(self.bottom or '0')
+        except ValueError:
+            bottom = 0
+
+        if left < 0:
+            left = 0
+
+        if left > 100:
+            left = 100
+
+        if top < 0:
+            top = 0
+
+        if top > 100:
+            top = 100
+
+        if right < 0:
+            right = 0
+
+        if right > 100:
+            right = 100
+
+        if bottom < 0:
+            bottom = 0
+
+        if bottom > 100:
+            bottom = 100
+
+        #if left > right:
+        #    left, right = right, left
+
+        #if top > bottom:
+        #    top, bottom = bottom, top
+
+        logger.debug(
+            'left: %f, top: %f, right: %f, bottom: %f', left, top, right,
+            bottom
+        )
+
+        fillcolor_value = getattr(self, 'fillcolor', None)
+        if fillcolor_value:
+            fill_color = ImageColor.getrgb(fillcolor_value)
+        else:
+            fill_color = 0
+
+        outlinecolor_value = getattr(self, 'outlinecolor', None)
+        if outlinecolor_value:
+            outline_color = ImageColor.getrgb(outlinecolor_value)
+        else:
+            outline_color = None
+
+        outlinewidth_value = getattr(self, 'outlinewidth', None)
+        if outlinewidth_value:
+            outline_width = int(outlinewidth_value)
+        else:
+            outline_width = 0
+
+        left = left / 100.0 * self.image.size[0]
+        top = top / 100.0 * self.image.size[1]
+
+        # Invert right value
+        # Pillow uses left, top, right, bottom to define a viewport
+        # of real coordinates
+        # We invert the right and bottom to define a viewport
+        # that can crop from the right and bottom borders without
+        # having to know the real dimensions of an image
+
+        right = self.image.size[0] - (right / 100.0 * self.image.size[0])
+        bottom = self.image.size[1] - (bottom / 100.0 * self.image.size[1])
+
+        draw = ImageDraw.Draw(self.image)
+        draw.rectangle(
+            (left, top, right, bottom), fill=fill_color, outline=outline_color,
+            width=outline_width
+        )
+
+        return self.image
+
+
 class TransformationFlip(BaseTransformation):
     arguments = ()
     label = _('Flip')
@@ -316,6 +524,8 @@ class TransformationZoom(BaseTransformation):
 
 
 BaseTransformation.register(transformation=TransformationCrop)
+BaseTransformation.register(transformation=TransformationDrawRectangle)
+BaseTransformation.register(transformation=TransformationDrawRectanglePercent)
 BaseTransformation.register(transformation=TransformationFlip)
 BaseTransformation.register(transformation=TransformationGaussianBlur)
 BaseTransformation.register(transformation=TransformationLineArt)

mayan/apps/document_states/apps.py

@@ -279,6 +279,8 @@ class DocumentStatesApp(MayanAppConfig):
         )
 
         SourceColumn(
+<<<<<<< HEAD
+=======
             attribute='name', is_identifier=True, is_sortable=True,
             source=WorkflowTransitionField
         )
@@ -303,6 +305,7 @@ class DocumentStatesApp(MayanAppConfig):
         )
 
         SourceColumn(
+>>>>>>> versions/minor
             source=WorkflowRuntimeProxy, label=_('Documents'),
             func=lambda context: context['object'].get_document_count(
                 user=context['request'].user

mayan/apps/documents/literals.py

@@ -32,6 +32,7 @@ DEFAULT_DOCUMENT_TYPE_LABEL = _('Default')
 DOCUMENT_IMAGE_TASK_TIMEOUT = 120
 STUB_EXPIRATION_INTERVAL = 60 * 60 * 24  # 24 hours
 UPDATE_PAGE_COUNT_RETRY_DELAY = 10
+UPLOAD_NEW_DOCUMENT_RETRY_DELAY = 10
 UPLOAD_NEW_VERSION_RETRY_DELAY = 10
 
 PAGE_RANGE_ALL = 'all'

mayan/apps/documents/queues.py

@@ -82,3 +82,7 @@ queue_uploads.add_task_type(
     dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for',
     label=_('Scan document duplicates')
 )
+queue_uploads.add_task_type(
+    dotted_path='mayan.apps.documents.tasks.task_upload_new_document',
+    label=_('Upload new document')
+)

mayan/apps/documents/tasks.py

@@ -9,7 +9,8 @@ from django.db import OperationalError
 from mayan.celery import app
 
 from .literals import (
-    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY
+    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY,
+    UPLOAD_NEW_VERSION_RETRY_DELAY
 )
 
 logger = logging.getLogger(__name__)
@@ -127,6 +128,60 @@ def task_update_page_count(self, version_id):
         raise self.retry(exc=exception)
 
 
+@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
+def task_upload_new_document(self, document_type_id, shared_uploaded_file_id):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+
+    SharedUploadedFile = apps.get_model(
+        app_label='common', model_name='SharedUploadedFile'
+    )
+
+    try:
+        document_type = DocumentType.objects.get(pk=document_type_id)
+        shared_file = SharedUploadedFile.objects.get(
+            pk=shared_uploaded_file_id
+        )
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to retrieve shared data for '
+            'new document of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+
+    try:
+        with shared_file.open() as file_object:
+            document_type.new_document(file_object=file_object)
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to create new document '
+            'of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+    except Exception as exception:
+        # This except and else block emulate a finally:
+        logger.error(
+            'Unexpected error during attempt to create new document '
+            'of type: %s; %s', document_type, exception
+        )
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+    else:
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+
+
 @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True)
 def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None):
     SharedUploadedFile = apps.get_model(

mayan/apps/importer/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'mayan.apps.importer.apps.ImporterApp'

mayan/apps/importer/apps.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.common.apps import MayanAppConfig
+
+
+class ImporterApp(MayanAppConfig):
+    app_namespace = 'importer'
+    app_url = 'importer'
+    has_rest_api = False
+    has_tests = True
+    name = 'mayan.apps.importer'
+    verbose_name = _('Importer')
+
+    def ready(self):
+        super(ImporterApp, self).ready()

mayan/apps/importer/management/commands/import.py

@@ -0,0 +1,150 @@
+from __future__ import unicode_literals
+
+import csv
+import time
+
+from django.apps import apps
+from django.core import management
+from django.core.files import File
+
+from ...tasks import task_upload_new_document
+
+
+class Command(management.BaseCommand):
+    help = 'Import documents from a CSV file.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--document_type_column',
+            action='store', dest='document_type_column', default=0,
+            help='Column that contains the document type labels. Column '
+            'numbers start at 0.',
+            type=int
+        )
+        parser.add_argument(
+            '--document_path_column',
+            action='store', dest='document_path_column', default=1,
+            help='Column that contains the path to the document files. Column '
+            'numbers start at 0.',
+            type=int
+        )
+        parser.add_argument(
+            '--ignore_errors',
+            action='store_true', dest='ignore_errors', default=False,
+            help='Don\'t stop the import process on common errors like '
+            'incorrect file paths.',
+        )
+        parser.add_argument(
+            '--ignore_rows',
+            action='store', dest='ignore_rows', default='',
+            help='Ignore a set of rows. Row numbers must be separated by commas.'
+        )
+        parser.add_argument(
+            '--metadata_pairs_column',
+            action='store', dest='metadata_pairs_column',
+            help='Column that contains metadata name and values for the '
+            'documents. Use the form: <label column>:<value column>. Example: '
+            '2:5. Separate multiple pairs with commas. Example: 2:5,7:10',
+        )
+        parser.add_argument('filelist', nargs='?', help='File list')
+
+    def handle(self, *args, **options):
+        time_start = time.time()
+        time_last_display = time_start
+        document_types = {}
+        uploaded_count = 0
+        row_count = 0
+        rows_to_ignore = []
+        for entry in options['ignore_rows'].split(','):
+            if entry:
+                rows_to_ignore.append(int(entry))
+
+        DocumentType = apps.get_model(
+            app_label='documents', model_name='DocumentType'
+        )
+        SharedUploadedFile = apps.get_model(
+            app_label='common', model_name='SharedUploadedFile'
+        )
+
+        if not options['filelist']:
+            self.stderr.write('Must specify a CSV file path.')
+            exit(1)
+        else:
+            with open(options['filelist']) as csv_datafile:
+                csv_reader = csv.reader(csv_datafile)
+                for row in csv_reader:
+                    # Increase row count here even though start index is 0
+                    # purpose is to avoid losing row number increments on
+                    # exceptions
+                    row_count = row_count + 1
+                    if row_count - 1 not in rows_to_ignore:
+                        try:
+                            with open(row[options['document_path_column']]) as file_object:
+                                document_type_label = row[options['document_type_column']]
+
+                                if document_type_label not in document_types:
+                                    self.stdout.write(
+                                        'New document type: {}. Creating and caching.'.format(
+                                            document_type_label
+                                        )
+                                    )
+                                    document_type, created = DocumentType.objects.get_or_create(
+                                        label=document_type_label
+                                    )
+                                    document_types[document_type_label] = document_type
+                                else:
+                                    document_type = document_types[document_type_label]
+
+                                shared_uploaded_file = SharedUploadedFile.objects.create(
+                                    file=File(file_object)
+                                )
+
+                                extra_data = {}
+                                if options['metadata_pairs_column']:
+                                    extra_data['metadata_pairs'] = []
+
+                                    for pair in options['metadata_pairs_column'].split(','):
+                                        name, value = pair.split(':')
+                                        extra_data['metadata_pairs'].append(
+                                            {
+                                                'name': row[int(name)],
+                                                'value': row[int(value)]
+                                            }
+                                        )
+
+                                task_upload_new_document.apply_async(
+                                    kwargs=dict(
+                                        document_type_id=document_type.pk,
+                                        shared_uploaded_file_id=shared_uploaded_file.pk,
+                                        extra_data=extra_data
+                                    )
+                                )
+
+                                uploaded_count = uploaded_count + 1
+
+                                if (time.time() - time_last_display) > 1:
+                                    time_last_display = time.time()
+                                    self.stdout.write(
+                                        'Time: {}s, Files copied and queued: {}, files processed per second: {}'.format(
+                                            int(time.time() - time_start),
+                                            uploaded_count,
+                                            uploaded_count / (time.time() - time_start)
+                                        )
+                                    )
+
+                        except (IOError, OSError) as exception:
+                            if not options['ignore_errors']:
+                                raise
+                            else:
+                                self.stderr.write(
+                                    'Error processing row: {}; {}.'.format(
+                                        row_count - 1, exception
+                                    )
+                                )
+
+            self.stdout.write(
+                'Total files copied and queues: {}'.format(uploaded_count)
+            )
+            self.stdout.write(
+                'Total time: {}'.format(time.time() - time_start)
+            )

mayan/apps/importer/queues.py

@@ -0,0 +1,10 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.documents.queues import queue_uploads
+
+queue_uploads.add_task_type(
+    dotted_path='mayan.apps.importer.tasks.task_upload_new_document',
+    label=_('Import new document')
+)

mayan/apps/importer/tasks.py

@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+import logging
+
+from django.apps import apps
+from django.db import OperationalError
+from django.utils.text import slugify
+
+from mayan.celery import app
+
+from mayan.apps.documents.literals import UPLOAD_NEW_DOCUMENT_RETRY_DELAY
+
+logger = logging.getLogger(__name__)
+
+
+@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
+def task_upload_new_document(self, document_type_id, shared_uploaded_file_id, extra_data=None):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+
+    MetadataType = apps.get_model(
+        app_label='metadata', model_name='MetadataType'
+    )
+
+    SharedUploadedFile = apps.get_model(
+        app_label='common', model_name='SharedUploadedFile'
+    )
+
+    try:
+        document_type = DocumentType.objects.get(pk=document_type_id)
+        shared_file = SharedUploadedFile.objects.get(
+            pk=shared_uploaded_file_id
+        )
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to retrieve shared data for '
+            'new document of type ID: %d; %s. Retrying.', document_type_id,
+            exception
+        )
+        raise self.retry(exc=exception)
+
+    try:
+        with shared_file.open() as file_object:
+            new_document = document_type.new_document(file_object=file_object)
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to create new document '
+            'of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+    except Exception as exception:
+        # This except and else block emulate a finally:
+        logger.error(
+            'Unexpected error during attempt to create new document '
+            'of type: %s; %s', document_type, exception
+        )
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+    else:
+        if extra_data:
+            for pair in extra_data.get('metadata_pairs', []):
+                name = slugify(pair['name']).replace('-', '_')
+                logger.debug(
+                    'Metadata pair (label, name, value): %s, %s, %s',
+                    pair['name'], name, pair['value']
+                )
+
+                metadata_type, created = MetadataType.objects.get_or_create(
+                    name=name, defaults={'label': pair['name']}
+                )
+                if not new_document.document_type.metadata.filter(metadata_type=metadata_type).exists():
+                    logger.debug('Metadata type created')
+                    new_document.document_type.metadata.create(
+                        metadata_type=metadata_type, required=False
+                    )
+
+                new_document.metadata.create(
+                    metadata_type=metadata_type, value=pair['value']
+                )
+
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )

mayan/apps/importer/tests/test_management_commands.py

@@ -0,0 +1,120 @@
+from __future__ import unicode_literals
+
+import csv
+
+from django.core import management
+from django.utils.encoding import force_bytes
+
+from mayan.apps.documents.models import DocumentType, Document
+from mayan.apps.documents.tests import GenericDocumentTestCase
+from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH
+from mayan.apps.storage.utils import fs_cleanup, mkstemp
+
+
+class ImportManagementCommandTestCase(GenericDocumentTestCase):
+    auto_generate_test_csv_file = True
+    auto_upload_document = False
+    random_primary_key_enable = False
+    test_import_count = 1
+
+    def setUp(self):
+        super(ImportManagementCommandTestCase, self).setUp()
+        if self.auto_generate_test_csv_file:
+            self._create_test_csv_file()
+
+    def tearDown(self):
+        self._destroy_test_csv_file()
+        super(ImportManagementCommandTestCase, self).tearDown()
+
+    def _create_test_csv_file(self):
+        self.test_csv_file_descriptor, self.test_csv_path = mkstemp()
+
+        print('Test CSV file: {}'.format(self.test_csv_path))
+
+        with open(self.test_csv_path, mode='wb') as csvfile:
+            filewriter = csv.writer(
+                csvfile, delimiter=force_bytes(','), quotechar=force_bytes('"'),
+                quoting=csv.QUOTE_MINIMAL
+            )
+            print(
+                'Generating test CSV for {} documents'.format(
+                    self.test_import_count
+                )
+            )
+            for times in range(self.test_import_count):
+                filewriter.writerow(
+                    [
+                        self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH,
+                        'column 2', 'column 3', 'column 4', 'column 5',
+                        'part #', 'value',
+                    ]
+                )
+                filewriter.writerow(
+                    [
+                        self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH,
+                        'column 2', 'column 3', 'column 4', 'column 5',
+                        'part#', 'value',
+                    ]
+                )
+
+    def _destroy_test_csv_file(self):
+        fs_cleanup(
+            filename=self.test_csv_path,
+            file_descriptor=self.test_csv_file_descriptor
+        )
+
+    def test_import_csv_read(self):
+        self.test_document_type.delete()
+        management.call_command('import', self.test_csv_path)
+
+        self.assertTrue(DocumentType.objects.count() > 0)
+        self.assertTrue(Document.objects.count() > 0)
+
+    def test_import_document_type_column_mapping(self):
+        self.test_document_type.delete()
+        management.call_command(
+            'import', self.test_csv_path, '--document_type_column', '2'
+        )
+
+        self.assertTrue(DocumentType.objects.first().label == 'column 2')
+        self.assertTrue(Document.objects.count() > 0)
+
+    def test_import_document_path_column_mapping(self):
+        self.test_document_type.delete()
+        with self.assertRaises(IOError):
+            management.call_command(
+                'import', self.test_csv_path, '--document_path_column', '2'
+            )
+
+    def test_import_metadata_column_mapping(self):
+        self.test_document_type.delete()
+        management.call_command(
+            'import', self.test_csv_path, '--metadata_pairs_column', '2:3,4:5',
+        )
+
+        self.assertTrue(DocumentType.objects.count() > 0)
+        self.assertTrue(Document.objects.count() > 0)
+        self.assertTrue(Document.objects.first().metadata.count() > 0)
+        self.assertEqual(
+            Document.objects.first().metadata.get(
+                metadata_type__name='column_2'
+            ).value, 'column 3'
+        )
+
+    def test_import_ambiguous_metadata(self):
+        self.auto_generate_test_csv_file = False
+        self.test_import_count = 2
+
+        self.test_document_type.delete()
+        management.call_command(
+            'import', self.test_csv_path, '--metadata_pairs_column', '6:7',
+        )
+
+        self.assertTrue(DocumentType.objects.count() > 0)
+        self.assertTrue(Document.objects.count() > 0)
+        self.assertTrue(Document.objects.first().metadata.count() > 0)
+        self.assertEqual(
+            Document.objects.first().metadata.get(
+                metadata_type__name='part'
+            ).value, 'value'
+        )

mayan/apps/navigation/classes.py

@@ -679,7 +679,7 @@ class SourceColumn(object):
                 logger.warning(
                     'No request variable, aborting request resolution'
                 )
-                return result
+                return final_result
 
         current_view_name = get_current_view_name(request=request)
         for column in columns:
@@ -692,7 +692,7 @@ class SourceColumn(object):
         return final_result
 
     def __init__(
-        self, source, attribute=None, empty_value=None, func=None,
+        self, source, attribute=None, empty_value=None, exclude=None, func=None,
         include_label=False, is_attribute_absolute_url=False,
         is_object_absolute_url=False, is_identifier=False, is_sortable=False,
         kwargs=None, label=None, order=None, sort_field=None, views=None,
@@ -702,7 +702,11 @@ class SourceColumn(object):
         self._label = label
         self.attribute = attribute
         self.empty_value = empty_value
+<<<<<<< HEAD
+        self.exclude = exclude or ()
+=======
         self.exclude = ()
+>>>>>>> versions/minor
         self.func = func
         self.is_attribute_absolute_url = is_attribute_absolute_url
         self.is_object_absolute_url = is_object_absolute_url

mayan/apps/navigation/templatetags/navigation_tags.py

@@ -113,12 +113,6 @@ def navigation_source_column_get_absolute_url(source_column, obj):
     return source_column.get_absolute_url(obj=obj)
 
 
-@register.simple_tag(takes_context=True)
-def resolve_link(context, link):
-    # This can be used to resolve links or menus too
-    return link.resolve(context=context)
-
-
 @register.simple_tag(takes_context=True)
 def navigation_source_column_resolve(context, column):
     if column:
@@ -126,3 +120,9 @@ def navigation_source_column_resolve(context, column):
         return result
     else:
         return ''
+
+
+@register.simple_tag(takes_context=True)
+def resolve_link(context, link):
+    # This can be used to resolve links or menus too
+    return link.resolve(context=context)

mayan/apps/redactions/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'mayan.apps.redactions.apps.RedactionsApp'

mayan/apps/redactions/apps.py

@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+import logging
+
+from django.apps import apps
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.common.apps import MayanAppConfig
+from mayan.apps.common.menus import (
+    menu_list_facet, menu_object, menu_secondary,
+)
+
+from .dependencies import *  # NOQA
+from .links import (
+    link_redaction_create, link_redaction_delete, link_redaction_edit,
+    link_redaction_list
+)
+
+logger = logging.getLogger(__name__)
+
+
+class RedactionsApp(MayanAppConfig):
+    app_namespace = 'redactions'
+    app_url = 'redactions'
+    has_rest_api = False
+    has_tests = False
+    name = 'mayan.apps.redactions'
+    verbose_name = _('Redactions')
+
+    def ready(self):
+        super(RedactionsApp, self).ready()
+
+        DocumentPage = apps.get_model(
+            app_label='documents', model_name='DocumentPage'
+        )
+        Redaction = self.get_model(model_name='Redaction')
+
+        menu_list_facet.bind_links(
+            links=(
+                link_redaction_list,
+            ), sources=(DocumentPage,)
+        )
+        menu_object.bind_links(
+            links=(link_redaction_delete, link_redaction_edit,),
+            sources=(Redaction,)
+        )
+        menu_secondary.bind_links(
+            links=(link_redaction_create,), sources=(Redaction,)
+        )
+        menu_secondary.bind_links(
+            links=(link_redaction_create,),
+            sources=(
+                'redactions:redaction_create',
+                'redactions:redaction_list'
+            )
+        )

mayan/apps/redactions/classes.py

@@ -0,0 +1,28 @@
+'''
+from __future__ import unicode_literals
+
+import logging
+
+from PIL import Image
+
+from converter import converter_class
+
+logger = logging.getLogger(__name__)
+
+
+class OCRBackendBase(object):
+    def execute(self, file_object, language=None, process_barcodes=True, process_text=True, transformations=None):
+        self.language = language
+        self.process_barcodes = process_barcodes
+        self.process_text = process_text
+
+        if not transformations:
+            transformations = []
+
+        self.converter = converter_class(file_object=file_object)
+
+        for transformation in transformations:
+            self.converter.transform(transformation=transformation)
+
+        self.image = Image.open(self.converter.get_page())
+'''

mayan/apps/redactions/dependencies.py

@@ -0,0 +1,15 @@
+from __future__ import unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.dependencies.classes import (
+    GoogleFontDependency, JavaScriptDependency
+)
+
+JavaScriptDependency(
+    label=_('JavaScript image cropper'), module=__name__, name='cropperjs',
+    version_string='=1.4.1'
+)
+JavaScriptDependency(
+    module=__name__, name='jquery-cropper', version_string='=1.0.0'
+)

mayan/apps/redactions/exceptions.py

@@ -0,0 +1,16 @@
+'''from __future__ import unicode_literals
+
+
+class OCRError(Exception):
+    """
+    Raised by the OCR backend
+    """
+    pass
+
+
+class UnpaperError(Exception):
+    """
+    Raised by unpaper
+    """
+    pass
+'''

mayan/apps/redactions/forms.py

@@ -0,0 +1,15 @@
+from __future__ import unicode_literals
+
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+from .models import Redaction
+
+
+class RedactionCoordinatesForm(forms.ModelForm):
+    class Meta:
+        fields = ('arguments',)
+        model = Redaction
+        widgets = {
+            'arguments': forms.widgets.Textarea(attrs={'class': 'hidden'}),
+        }

mayan/apps/redactions/icons.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import, unicode_literals
+
+from mayan.apps.appearance.classes import Icon
+
+icon_redaction_create = Icon(
+    driver_name='fontawesome-dual', primary_symbol='highlighter',
+    secondary_symbol='plus'
+)
+icon_redaction_delete = Icon(driver_name='fontawesome', symbol='times')
+icon_redaction_edit = Icon(driver_name='fontawesome', symbol='pencil-alt')
+icon_redactions = Icon(driver_name='fontawesome', symbol='highlighter')

mayan/apps/redactions/links.py

@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.navigation.classes import Link
+
+from .permissions import (
+    permission_redaction_create, permission_redaction_delete,
+    permission_redaction_edit, permission_redaction_view
+)
+
+link_redaction_create = Link(
+    icon_class_path='mayan.apps.redactions.icons.icon_redaction_create',
+    permissions=(permission_redaction_create,), text=_('Create redaction'),
+    view='redactions:redaction_create', args='resolved_object.id'
+)
+link_redaction_delete = Link(
+    icon_class_path='mayan.apps.redactions.icons.icon_redaction_delete',
+    permissions=(permission_redaction_delete,), tags='dangerous',
+    text=_('Delete'), view='redactions:redaction_delete',
+    args='resolved_object.id'
+)
+link_redaction_edit = Link(
+    icon_class_path='mayan.apps.redactions.icons.icon_redaction_edit',
+    permissions=(permission_redaction_edit,), text=_('Edit'),
+    view='redactions:redaction_edit', args='resolved_object.id'
+)
+link_redaction_list = Link(
+    icon_class_path='mayan.apps.redactions.icons.icon_redactions',
+    permissions=(permission_redaction_view,), text=_('Redactions'),
+    view='redactions:redaction_list', args='resolved_object.id'
+)

mayan/apps/redactions/migrations/0001_initial.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-26 19:04
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('converter', '0014_auto_20190626_1904'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Redaction',
+            fields=[
+            ],
+            options={
+                'verbose_name': 'Redaction',
+                'proxy': True,
+                'verbose_name_plural': 'Redactions',
+                'indexes': [],
+            },
+            bases=('converter.transformation',),
+        ),
+    ]

mayan/apps/redactions/models.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.converter.models import Transformation
+
+
+class Redaction(Transformation):
+    class Meta:
+        proxy = True
+        verbose_name = _('Redaction')
+        verbose_name_plural = _('Redactions')

mayan/apps/redactions/permissions.py

@@ -0,0 +1,20 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.permissions import PermissionNamespace
+
+namespace = PermissionNamespace(label=_('Redactions'), name='redactions')
+
+permission_redaction_create = namespace.add_permission(
+    label=_('Create new redactions'), name='redaction_create'
+)
+permission_redaction_delete = namespace.add_permission(
+    label=_('Delete redactions'), name='redaction_delete'
+)
+permission_redaction_edit = namespace.add_permission(
+    label=_('Edit redactions'), name='redaction_edit'
+)
+permission_redaction_view = namespace.add_permission(
+    label=_('View existing redactions'), name='redaction_view'
+)

mayan/apps/redactions/serializers.py

@@ -0,0 +1,5 @@
+from rest_framework import serializers
+
+
+class DocumentVersionOCRSerializer(serializers.Serializer):
+    document_version_id = serializers.IntegerField()

mayan/apps/redactions/templates/redactions/cropper.html

@@ -0,0 +1,113 @@
+{% extends 'appearance/base.html' %}
+
+{% load i18n %}
+{% load static %}
+
+{% load common_tags %}
+
+{% block title %}{% blocktrans with object as object %}Redaction coordinates for: {{ object }}{% endblocktrans %}{% endblock title %}
+
+{% block stylesheets %}
+    <link href="{% static 'redactions/node_modules/cropperjs/dist/cropper.css' %}" rel="stylesheet">
+
+    <style>
+        .cropper-main {
+            width: 100%;
+        }
+
+        .cropper-main img {
+            max-width: 100%;
+        }
+    </style>
+{% endblock %}
+
+{% block content %}
+    <div class="cropper-main">
+        <img src="{{ document_page.get_api_image_url }}">
+    </div>
+
+    <br>
+    {% with '' as title %}
+        {% include 'appearance/generic_form_subtemplate.html' %}
+    {% endwith %}
+{% endblock content %}
+
+{% block javascript %}
+    <script>
+        var crop_left, crop_top, crop_right, crop_bottom;
+        var pic_real_width, pic_real_height;
+        var canvasData;
+        var containerData;
+        var $image = $('.cropper-main img');
+        var cropperInstance;
+        var defaultArguments = {
+            left: 10,
+            top: 10,
+            right: 10,
+            bottom: 10,
+            fillcolor: '#000000',
+        }
+        var initialArguments = JSON.parse($('#id_arguments').text() || JSON.stringify(defaultArguments));
+
+        var callbackCrop = function (data) {
+            var crop_left = (data.detail.x / pic_real_width * 100).toFixed(2);
+            var crop_top = (data.detail.y / pic_real_height * 100).toFixed(2);
+            var crop_right = (100.001 - (data.detail.x + data.detail.width) / pic_real_width * 100).toFixed(2);
+            var crop_bottom = (100.001 - (data.detail.y + data.detail.height) / pic_real_height * 100).toFixed(2);
+
+            var arguments = {
+                'left': parseFloat(crop_left),
+                'top': parseFloat(crop_top),
+                'right': parseFloat(crop_right),
+                'bottom': parseFloat(crop_bottom),
+                'fillcolor': '#000000',
+            }
+
+            $('#id_arguments').text(JSON.stringify(arguments));
+        }
+
+        jQuery(document).ready(function() {
+            $('.help-block').hide();
+            $('label').hide();
+        });
+
+        $.getScript("{% static 'redactions/node_modules/cropperjs/dist/cropper.js' %}")
+        .done(function (script, textStatus) {
+            $.getScript("{% static 'redactions/node_modules/jquery-cropper/dist/jquery-cropper.js' %}")
+            .done(function (script, textStatus) {
+                jQuery(document).ready(function () {
+                    // Create DOM new image to get the real
+                    // (unscaled) image size
+                    $('<img/>')
+                    .attr('src', $image.attr('src'))
+                    .on('load', function () {
+                        pic_real_width = this.width;
+                        pic_real_height = this.height;
+                    });
+
+                    cropperInstance = $image.cropper({
+                        crop: callbackCrop,
+                        mouseWheelZoom: false,
+                        movable: false,
+                        //preview: '.cropper-preview',
+                        ready: function () {
+                            canvasData = $image.cropper('getCanvasData');
+                            containerData = $image.cropper('getContainerData');
+
+                            $image.cropper('setCropBoxData', {
+                                left: initialArguments.left / 100.0 * canvasData.width + canvasData.left,
+                                top: initialArguments.top / 100.0 * canvasData.height + canvasData.top,
+                                width: (100.0 - initialArguments.right - initialArguments.left) / 100.0 * canvasData.width,
+                                height: (100.0 - initialArguments.bottom - initialArguments.top) / 100.0 * canvasData.height,
+                            });
+                        },
+                        rotatable: false,
+                        touchDragZoom: false,
+                        viewMode: 1,
+                        zoomable: false,
+                    });
+                })
+            })
+        });
+    </script>
+{% endblock %}

mayan/apps/redactions/urls.py

@@ -0,0 +1,30 @@
+from __future__ import unicode_literals
+
+from django.conf.urls import url
+
+from .views import (
+    RedactionCreateView, RedactionDeleteView, RedactionEditView,
+    RedactionListView,
+)
+
+
+urlpatterns = [
+    url(
+        regex=r'^document_pages/(?P<pk>\d+)/redactions/create/$',
+        view=RedactionCreateView.as_view(), name='redaction_create'
+    ),
+    url(
+        regex=r'^document_pages/(?P<pk>\d+)/redactions/$',
+        view=RedactionListView.as_view(), name='redaction_list'
+    ),
+    url(
+        regex=r'^redactions/(?P<pk>\d+)/delete/$',
+        view=RedactionDeleteView.as_view(), name='redaction_delete'
+    ),
+    url(
+        regex=r'^redactions/(?P<pk>\d+)/edit/$',
+        view=RedactionEditView.as_view(), name='redaction_edit'
+    ),
+]
+
+api_urls = []

mayan/apps/redactions/views.py

@@ -0,0 +1,147 @@
+from __future__ import absolute_import, unicode_literals
+
+import logging
+
+from django.core.urlresolvers import reverse
+from django.template import RequestContext
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.common.generics import (
+    SingleObjectCreateView, SingleObjectDeleteView, SingleObjectEditView,
+    SingleObjectListView
+)
+from mayan.apps.common.mixins import ExternalObjectMixin
+from mayan.apps.converter.transformations import TransformationDrawRectanglePercent
+from mayan.apps.documents.models import DocumentPage
+
+from .forms import RedactionCoordinatesForm
+from .icons import icon_redactions
+from .links import link_redaction_create
+from .models import Redaction
+from .permissions import (
+    permission_redaction_create, permission_redaction_delete,
+    permission_redaction_edit, permission_redaction_view
+)
+
+logger = logging.getLogger(__name__)
+
+
+class RedactionCreateView(ExternalObjectMixin, SingleObjectCreateView):
+    external_object_class = DocumentPage
+    external_object_pk_url_kwarg = 'pk'
+    form_class = RedactionCoordinatesForm
+    model = Redaction
+    object_permission = permission_redaction_create
+    template_name = 'redactions/cropper.html'
+
+    def form_valid(self, form):
+        instance = form.save(commit=False)
+        instance.content_object = self.external_object
+        instance.name = TransformationDrawRectanglePercent.name
+        instance.save()
+        return super(RedactionCreateView, self).form_valid(form)
+
+    def get_extra_context(self, **kwargs):
+        context = {
+            'document_page': self.external_object,
+            'redaction': self.object,
+            'title': _('Create redaction for: %s') % self.external_object
+        }
+
+        return context
+
+    def get_post_action_redirect(self):
+        return reverse(
+            viewname='redactions:redaction_list', kwargs={
+                'pk': self.external_object.pk
+            }
+        )
+
+
+class RedactionDeleteView(SingleObjectDeleteView):
+    model = Redaction
+    object_permission = permission_redaction_delete
+
+    def get_post_action_redirect(self):
+        return reverse(
+            viewname='redactions:redaction_list', kwargs={
+                'pk': self.object.content_object.pk
+            }
+        )
+
+    def get_extra_context(self):
+        return {
+            'content_object': self.object.content_object,
+            'navigation_object_list': ('content_object', 'redaction'),
+            'previous': reverse(
+                viewname='redactions:redaction_list', kwargs={
+                    'pk': self.object.content_object.pk
+                }
+            ),
+            'redaction': self.object,
+            'title': _(
+                'Delete refaction for: %(content_object)s?'
+            ) % {
+                'content_object': self.object.content_object
+            },
+        }
+
+
+class RedactionEditView(SingleObjectEditView):
+    form_class = RedactionCoordinatesForm
+    model = Redaction
+    object_permission = permission_redaction_edit
+    template_name = 'redactions/cropper.html'
+
+    def get_extra_context(self, **kwargs):
+        context = {
+            'document_page': self.object.content_object,
+            'navigation_object_list': ['document_page', 'redaction'],
+            'redaction': self.object,
+            'title': _('Edit redaction: %s') % self.object
+        }
+
+        return context
+
+    def get_post_action_redirect(self):
+        return reverse(
+            viewname='redactions:redaction_list', kwargs={
+                'pk': self.object.content_object.pk
+            }
+        )
+
+
+class RedactionListView(ExternalObjectMixin, SingleObjectListView):
+    external_object_class = DocumentPage
+    object_permission = permission_redaction_view
+    external_object_pk_url_kwarg = 'pk'
+
+    def dispatch(self, request, *args, **kwargs):
+        return super(RedactionListView, self).dispatch(
+            request, *args, **kwargs
+        )
+
+    def get_extra_context(self):
+        return {
+            'hide_object': True,
+            'object': self.external_object,
+            'no_results_icon': icon_redactions,
+            'no_results_main_link': link_redaction_create.resolve(
+                context=RequestContext(
+                    request=self.request, dict_={
+                        'object': self.external_object
+                    }
+                )
+            ),
+            'no_results_text': _(
+                'Redactions allow removing access to confidential and '
+                'sensitive information without having to modify the document.'
+            ),
+            'no_results_title': _('No existing redactions'),
+            'title': _('Redactions for: %s') % self.external_object,
+        }
+
+    def get_source_queryset(self):
+        return Redaction.objects.get_for_object(
+            obj=self.external_object
+        ).filter(name__startswith='draw')

mayan/settings/base.py

@@ -116,11 +116,13 @@ INSTALLED_APPS = (
     'mayan.apps.document_states',
     'mayan.apps.documents',
     'mayan.apps.file_metadata',
+    'mayan.apps.importer',
     'mayan.apps.linking',
     'mayan.apps.mailer',
     'mayan.apps.mayan_statistics',
     'mayan.apps.metadata',
     'mayan.apps.mirroring',
+    'mayan.apps.redactions',
     'mayan.apps.ocr',
     'mayan.apps.sources',
     'mayan.apps.storage',