From b3d59eee39677632f2025488517b7c2e1885d06d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 19 Jun 2019 16:02:00 -0400 Subject: [PATCH] Add MVP of the importer app Signed-off-by: Roberto Rosario --- HISTORY.rst | 3 +- mayan/apps/importer/__init__.py | 3 + mayan/apps/importer/apps.py | 17 ++++ mayan/apps/importer/management/__init__.py | 0 .../importer/management/commands/__init__.py | 0 .../importer/management/commands/import.py | 82 +++++++++++++++++++ mayan/apps/importer/tests/__init__.py | 0 .../tests/test_management_commands.py | 58 +++++++++++++ mayan/settings/base.py | 1 + 9 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 mayan/apps/importer/__init__.py create mode 100644 mayan/apps/importer/apps.py create mode 100644 mayan/apps/importer/management/__init__.py create mode 100644 mayan/apps/importer/management/commands/__init__.py create mode 100644 mayan/apps/importer/management/commands/import.py create mode 100644 mayan/apps/importer/tests/__init__.py create mode 100644 mayan/apps/importer/tests/test_management_commands.py diff --git a/HISTORY.rst b/HISTORY.rst index dc473e2b00..361dacf353 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,8 @@ * Add support for disabling the random primary key test mixin. * Add a reusable task to upload documents. - +* Add MVP of the importer app. + 3.2.2 (2019-06-19) ================== * Fix document type change view. Closes GitLab issue #614 diff --git a/mayan/apps/importer/__init__.py b/mayan/apps/importer/__init__.py new file mode 100644 index 0000000000..11a51a9d8e --- /dev/null +++ b/mayan/apps/importer/__init__.py @@ -0,0 +1,3 @@ +from __future__ import unicode_literals + +default_app_config = 'mayan.apps.importer.apps.ImporterApp' diff --git a/mayan/apps/importer/apps.py b/mayan/apps/importer/apps.py new file mode 100644 index 0000000000..59b7a5b4ef --- /dev/null +++ b/mayan/apps/importer/apps.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.apps import MayanAppConfig + + +class ImporterApp(MayanAppConfig): + app_namespace = 'importer' + app_url = 'importer' + has_rest_api = False + has_tests = True + name = 'mayan.apps.importer' + verbose_name = _('Importer') + + def ready(self): + super(ImporterApp, self).ready() diff --git a/mayan/apps/importer/management/__init__.py b/mayan/apps/importer/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/importer/management/commands/__init__.py b/mayan/apps/importer/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/importer/management/commands/import.py b/mayan/apps/importer/management/commands/import.py new file mode 100644 index 0000000000..3ae3ca0813 --- /dev/null +++ b/mayan/apps/importer/management/commands/import.py @@ -0,0 +1,82 @@ +from __future__ import unicode_literals + +import csv +import time + +from django.apps import apps +from django.core import management +from django.core.files import File + +from mayan.apps.documents.tasks import task_upload_new_document + + +class Command(management.BaseCommand): + help = 'Import documents from a CSV file.' + + def add_arguments(self, parser): + #parser.add_argument( + # '-l', '--link', + # action='store_true', dest='link', default=False, + # help='Create a symbolic link to each file instead of copying.', + #) + parser.add_argument('filelist', nargs='?', help='File list') + + def handle(self, *args, **options): + time_start = time.time() + time_last_display = time_start + document_types = {} + uploaded_count = 0 + + DocumentType = apps.get_model( + app_label='documents', model_name='DocumentType' + ) + SharedUploadedFile = apps.get_model( + app_label='common', model_name='SharedUploadedFile' + ) + + if not options['filelist']: + self.stderr.write('Must specify a CSV file path.') + exit(1) + else: + with open(options['filelist']) as csv_datafile: + csv_reader = csv.reader(csv_datafile) + for row in csv_reader: + with open(row[1]) as file_object: + if row[0] not in document_types: + self.stdout.write('New document type: {}. Creating and caching.'.format(row[0])) + document_type, created = DocumentType.objects.get_or_create( + label=row[0] + ) + document_types[row[0]] = document_type + else: + document_type = document_types[row[0]] + + shared_uploaded_file = SharedUploadedFile.objects.create( + file=File(file_object) + ) + + task_upload_new_document.apply_async( + kwargs=dict( + document_type_id=document_type.pk, + shared_uploaded_file_id=shared_uploaded_file.pk, + ) + ) + + uploaded_count = uploaded_count + 1 + + if (time.time() - time_last_display) > 1: + time_last_display = time.time() + self.stdout.write( + 'Time: {}s, Files copied and queued: {}, files processed per second: {}'.format( + int(time.time() - time_start), + uploaded_count, + uploaded_count / (time.time() - time_start) + ) + ) + + self.stdout.write( + 'Total files copied and queues: {}'.format(uploaded_count) + ) + self.stdout.write( + 'Total time: {}'.format(time.time() - time_start) + ) diff --git a/mayan/apps/importer/tests/__init__.py b/mayan/apps/importer/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/importer/tests/test_management_commands.py b/mayan/apps/importer/tests/test_management_commands.py new file mode 100644 index 0000000000..8853e7d2b5 --- /dev/null +++ b/mayan/apps/importer/tests/test_management_commands.py @@ -0,0 +1,58 @@ +from __future__ import unicode_literals + +import csv + +from django.core import management +from django.utils.encoding import force_bytes + +from mayan.apps.documents.models import DocumentType, Document +from mayan.apps.documents.tests import GenericDocumentTestCase +from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH +from mayan.apps.storage.utils import fs_cleanup, mkstemp + + +class ImportManagementCommandTestCase(GenericDocumentTestCase): + auto_upload_document = False + random_primary_key_enable = False + test_import_count = 1 + + def setUp(self): + super(ImportManagementCommandTestCase, self).setUp() + self._create_test_csv_file() + + def tearDown(self): + self._destroy_test_csv_file() + super(ImportManagementCommandTestCase, self).tearDown() + + def _create_test_csv_file(self): + self.test_csv_file_descriptor, self.test_csv_path = mkstemp() + + print('Test CSV file: {}'.format(self.test_csv_path)) + + with open(self.test_csv_path, mode='wb') as csvfile: + filewriter = csv.writer( + csvfile, delimiter=force_bytes(','), quotechar=force_bytes('"'), + quoting=csv.QUOTE_MINIMAL + ) + print( + 'Generating test CSV for {} documents'.format( + self.test_import_count + ) + ) + for times in range(self.test_import_count): + filewriter.writerow( + [self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH] + ) + + def _destroy_test_csv_file(self): + fs_cleanup( + filename=self.test_csv_path, + file_descriptor=self.test_csv_file_descriptor + ) + + def test_import_csv_read(self): + self.test_document_type.delete() + management.call_command('import', self.test_csv_path) + + self.assertTrue(DocumentType.objects.count() > 0) + self.assertTrue(Document.objects.count() > 0) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index f2cd8f833b..349e5ab34f 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -120,6 +120,7 @@ INSTALLED_APPS = ( 'mayan.apps.document_states', 'mayan.apps.documents', 'mayan.apps.file_metadata', + 'mayan.apps.importer', 'mayan.apps.linking', 'mayan.apps.mailer', 'mayan.apps.mayan_statistics',