Add MVP of the importer app

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -3,7 +3,8 @@
 * Add support for disabling the random primary key
   test mixin.
 * Add a reusable task to upload documents.
- 
+* Add MVP of the importer app. 
+
 3.2.2 (2019-06-19)
 ==================
 * Fix document type change view. Closes GitLab issue #614

mayan/apps/importer/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'mayan.apps.importer.apps.ImporterApp'

mayan/apps/importer/apps.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.common.apps import MayanAppConfig
+
+
+class ImporterApp(MayanAppConfig):
+    app_namespace = 'importer'
+    app_url = 'importer'
+    has_rest_api = False
+    has_tests = True
+    name = 'mayan.apps.importer'
+    verbose_name = _('Importer')
+
+    def ready(self):
+        super(ImporterApp, self).ready()

mayan/apps/importer/management/commands/import.py

@@ -0,0 +1,82 @@
+from __future__ import unicode_literals
+
+import csv
+import time
+
+from django.apps import apps
+from django.core import management
+from django.core.files import File
+
+from mayan.apps.documents.tasks import task_upload_new_document
+
+
+class Command(management.BaseCommand):
+    help = 'Import documents from a CSV file.'
+
+    def add_arguments(self, parser):
+        #parser.add_argument(
+        #    '-l', '--link',
+        #    action='store_true', dest='link', default=False,
+        #    help='Create a symbolic link to each file instead of copying.',
+        #)
+        parser.add_argument('filelist', nargs='?', help='File list')
+
+    def handle(self, *args, **options):
+        time_start = time.time()
+        time_last_display = time_start
+        document_types = {}
+        uploaded_count = 0
+
+        DocumentType = apps.get_model(
+            app_label='documents', model_name='DocumentType'
+        )
+        SharedUploadedFile = apps.get_model(
+            app_label='common', model_name='SharedUploadedFile'
+        )
+
+        if not options['filelist']:
+            self.stderr.write('Must specify a CSV file path.')
+            exit(1)
+        else:
+            with open(options['filelist']) as csv_datafile:
+                csv_reader = csv.reader(csv_datafile)
+                for row in csv_reader:
+                    with open(row[1]) as file_object:
+                        if row[0] not in document_types:
+                            self.stdout.write('New document type: {}. Creating and caching.'.format(row[0]))
+                            document_type, created = DocumentType.objects.get_or_create(
+                                label=row[0]
+                            )
+                            document_types[row[0]] = document_type
+                        else:
+                            document_type = document_types[row[0]]
+
+                        shared_uploaded_file = SharedUploadedFile.objects.create(
+                            file=File(file_object)
+                        )
+
+                        task_upload_new_document.apply_async(
+                            kwargs=dict(
+                                document_type_id=document_type.pk,
+                                shared_uploaded_file_id=shared_uploaded_file.pk,
+                            )
+                        )
+
+                        uploaded_count = uploaded_count + 1
+
+                        if (time.time() - time_last_display) > 1:
+                            time_last_display = time.time()
+                            self.stdout.write(
+                                'Time: {}s, Files copied and queued: {}, files processed per second: {}'.format(
+                                    int(time.time() - time_start),
+                                    uploaded_count,
+                                    uploaded_count / (time.time() - time_start)
+                                )
+                            )
+
+            self.stdout.write(
+                'Total files copied and queues: {}'.format(uploaded_count)
+            )
+            self.stdout.write(
+                'Total time: {}'.format(time.time() - time_start)
+            )

mayan/apps/importer/tests/test_management_commands.py

@@ -0,0 +1,58 @@
+from __future__ import unicode_literals
+
+import csv
+
+from django.core import management
+from django.utils.encoding import force_bytes
+
+from mayan.apps.documents.models import DocumentType, Document
+from mayan.apps.documents.tests import GenericDocumentTestCase
+from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH
+from mayan.apps.storage.utils import fs_cleanup, mkstemp
+
+
+class ImportManagementCommandTestCase(GenericDocumentTestCase):
+    auto_upload_document = False
+    random_primary_key_enable = False
+    test_import_count = 1
+
+    def setUp(self):
+        super(ImportManagementCommandTestCase, self).setUp()
+        self._create_test_csv_file()
+
+    def tearDown(self):
+        self._destroy_test_csv_file()
+        super(ImportManagementCommandTestCase, self).tearDown()
+
+    def _create_test_csv_file(self):
+        self.test_csv_file_descriptor, self.test_csv_path = mkstemp()
+
+        print('Test CSV file: {}'.format(self.test_csv_path))
+
+        with open(self.test_csv_path, mode='wb') as csvfile:
+            filewriter = csv.writer(
+                csvfile, delimiter=force_bytes(','), quotechar=force_bytes('"'),
+                quoting=csv.QUOTE_MINIMAL
+            )
+            print(
+                'Generating test CSV for {} documents'.format(
+                    self.test_import_count
+                )
+            )
+            for times in range(self.test_import_count):
+                filewriter.writerow(
+                    [self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH]
+                )
+
+    def _destroy_test_csv_file(self):
+        fs_cleanup(
+            filename=self.test_csv_path,
+            file_descriptor=self.test_csv_file_descriptor
+        )
+
+    def test_import_csv_read(self):
+        self.test_document_type.delete()
+        management.call_command('import', self.test_csv_path)
+
+        self.assertTrue(DocumentType.objects.count() > 0)
+        self.assertTrue(Document.objects.count() > 0)

mayan/settings/base.py

@@ -120,6 +120,7 @@ INSTALLED_APPS = (
     'mayan.apps.document_states',
     'mayan.apps.documents',
     'mayan.apps.file_metadata',
+    'mayan.apps.importer',
     'mayan.apps.linking',
     'mayan.apps.mailer',
     'mayan.apps.mayan_statistics',