Compare commits

..

2 Commits

Author SHA1 Message Date
Roberto Rosario
adf47646bf Initial commit for the document trashed event
Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>
2019-06-24 08:29:45 -04:00
Roberto Rosario
4d7c0552bd Fix help text of the platformtemplate command
Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>
2019-06-22 00:04:34 -04:00
22 changed files with 116 additions and 515 deletions

View File

@@ -1,19 +1,13 @@
Importer branch
===============
* Add a reusable task to upload documents.
* Add MVP of the importer app.
3.2.4 (2019-06-XX) 3.2.4 (2019-06-XX)
================== ==================
* Support configurable GUnicorn timeouts. Defaults to * Support configurable GUnicorn timeouts. Defaults to
current value of 120 seconds. current value of 120 seconds.
* Fix help text of the platformtemplate command.
3.2.3 (2019-06-21) 3.2.3 (2019-06-21)
================== ==================
* Add support for disabling the random primary key * Add support for disabling the random primary key
test mixin. test mixin.
* Add a reusable task to upload documents.
* Add MVP of the importer app.
* Fix mailing profile log columns mappings. * Fix mailing profile log columns mappings.
GitLab issue #626. Thanks to Jesaja Everling (@jeverling) GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
for the report. for the report.

View File

@@ -19,6 +19,7 @@ Changes
GitLab issue #625. Thanks to Jesaja Everling (@jeverling) GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
for the report and the research. for the report and the research.
Removals Removals
-------- --------

View File

@@ -9,7 +9,7 @@ Changes
- Support configurable GUnicorn timeouts. Defaults to - Support configurable GUnicorn timeouts. Defaults to
current value of 120 seconds. current value of 120 seconds.
- Fix help text of the platformtemplate command.
Removals Removals
-------- --------

View File

@@ -18,6 +18,9 @@ event_document_new_version = namespace.add_event_type(
event_document_properties_edit = namespace.add_event_type( event_document_properties_edit = namespace.add_event_type(
label=_('Document properties edited'), name='document_edit' label=_('Document properties edited'), name='document_edit'
) )
event_document_trashed = namespace.add_event_type(
label=_('Document moved to trash'), name='document_trashed'
)
# The type of an existing document is changed to another type # The type of an existing document is changed to another type
event_document_type_change = namespace.add_event_type( event_document_type_change = namespace.add_event_type(
label=_('Document type changed'), name='document_type_change' label=_('Document type changed'), name='document_type_change'

View File

@@ -32,7 +32,6 @@ DEFAULT_DOCUMENT_TYPE_LABEL = _('Default')
DOCUMENT_IMAGE_TASK_TIMEOUT = 120 DOCUMENT_IMAGE_TASK_TIMEOUT = 120
STUB_EXPIRATION_INTERVAL = 60 * 60 * 24 # 24 hours STUB_EXPIRATION_INTERVAL = 60 * 60 * 24 # 24 hours
UPDATE_PAGE_COUNT_RETRY_DELAY = 10 UPDATE_PAGE_COUNT_RETRY_DELAY = 10
UPLOAD_NEW_DOCUMENT_RETRY_DELAY = 10
UPLOAD_NEW_VERSION_RETRY_DELAY = 10 UPLOAD_NEW_VERSION_RETRY_DELAY = 10
PAGE_RANGE_ALL = 'all' PAGE_RANGE_ALL = 'all'

View File

@@ -5,7 +5,7 @@ import uuid
from django.apps import apps from django.apps import apps
from django.core.files import File from django.core.files import File
from django.db import models from django.db import models, transaction
from django.urls import reverse from django.urls import reverse
from django.utils.encoding import python_2_unicode_compatible from django.utils.encoding import python_2_unicode_compatible
from django.utils.timezone import now from django.utils.timezone import now
@@ -13,7 +13,7 @@ from django.utils.translation import ugettext, ugettext_lazy as _
from ..events import ( from ..events import (
event_document_create, event_document_properties_edit, event_document_create, event_document_properties_edit,
event_document_type_change, event_document_trashed, event_document_type_change,
) )
from ..managers import DocumentManager, PassthroughManager, TrashCanManager from ..managers import DocumentManager, PassthroughManager, TrashCanManager
from ..settings import setting_language from ..settings import setting_language
@@ -104,11 +104,14 @@ class Document(models.Model):
def delete(self, *args, **kwargs): def delete(self, *args, **kwargs):
to_trash = kwargs.pop('to_trash', True) to_trash = kwargs.pop('to_trash', True)
_user = kwargs.pop('_user', None)
if not self.in_trash and to_trash: if not self.in_trash and to_trash:
self.in_trash = True with transaction.atomic():
self.deleted_date_time = now() self.in_trash = True
self.save() self.deleted_date_time = now()
self.save()
event_document_trashed.commit(actor=_user, target=self)
else: else:
for version in self.versions.all(): for version in self.versions.all():
version.delete() version.delete()
@@ -177,21 +180,23 @@ class Document(models.Model):
user = kwargs.pop('_user', None) user = kwargs.pop('_user', None)
_commit_events = kwargs.pop('_commit_events', True) _commit_events = kwargs.pop('_commit_events', True)
new_document = not self.pk new_document = not self.pk
super(Document, self).save(*args, **kwargs)
if new_document: with transaction.atomic():
if user: super(Document, self).save(*args, **kwargs)
self.add_as_recent_document_for_user(user)
event_document_create.commit( if new_document:
actor=user, target=self, action_object=self.document_type if user:
) self.add_as_recent_document_for_user(user)
event_document_create.commit(
actor=user, target=self, action_object=self.document_type
)
else:
event_document_create.commit(
target=self, action_object=self.document_type
)
else: else:
event_document_create.commit( if _commit_events:
target=self, action_object=self.document_type event_document_properties_edit.commit(actor=user, target=self)
)
else:
if _commit_events:
event_document_properties_edit.commit(actor=user, target=self)
def save_to_file(self, *args, **kwargs): def save_to_file(self, *args, **kwargs):
return self.latest_version.save_to_file(*args, **kwargs) return self.latest_version.save_to_file(*args, **kwargs)
@@ -200,15 +205,16 @@ class Document(models.Model):
has_changed = self.document_type != document_type has_changed = self.document_type != document_type
self.document_type = document_type self.document_type = document_type
self.save() with transaction.atomic():
if has_changed or force: self.save()
post_document_type_change.send( if has_changed or force:
sender=self.__class__, instance=self post_document_type_change.send(
) sender=self.__class__, instance=self
)
event_document_type_change.commit(actor=_user, target=self) event_document_type_change.commit(actor=_user, target=self)
if _user: if _user:
self.add_as_recent_document_for_user(user=_user) self.add_as_recent_document_for_user(user=_user)
@property @property
def size(self): def size(self):

View File

@@ -82,7 +82,3 @@ queue_uploads.add_task_type(
dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for', dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for',
label=_('Scan document duplicates') label=_('Scan document duplicates')
) )
queue_uploads.add_task_type(
dotted_path='mayan.apps.documents.tasks.task_upload_new_document',
label=_('Upload new document')
)

View File

@@ -9,8 +9,7 @@ from django.db import OperationalError
from mayan.celery import app from mayan.celery import app
from .literals import ( from .literals import (
UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY, UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY
UPLOAD_NEW_VERSION_RETRY_DELAY
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -128,60 +127,6 @@ def task_update_page_count(self, version_id):
raise self.retry(exc=exception) raise self.retry(exc=exception)
@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
def task_upload_new_document(self, document_type_id, shared_uploaded_file_id):
DocumentType = apps.get_model(
app_label='documents', model_name='DocumentType'
)
SharedUploadedFile = apps.get_model(
app_label='common', model_name='SharedUploadedFile'
)
try:
document_type = DocumentType.objects.get(pk=document_type_id)
shared_file = SharedUploadedFile.objects.get(
pk=shared_uploaded_file_id
)
except OperationalError as exception:
logger.warning(
'Operational error during attempt to retrieve shared data for '
'new document of type: %s; %s. Retrying.', document_type, exception
)
raise self.retry(exc=exception)
try:
with shared_file.open() as file_object:
document_type.new_document(file_object=file_object)
except OperationalError as exception:
logger.warning(
'Operational error during attempt to create new document '
'of type: %s; %s. Retrying.', document_type, exception
)
raise self.retry(exc=exception)
except Exception as exception:
# This except and else block emulate a finally:
logger.error(
'Unexpected error during attempt to create new document '
'of type: %s; %s', document_type, exception
)
try:
shared_file.delete()
except OperationalError as exception:
logger.warning(
'Operational error during attempt to delete shared '
'file: %s; %s.', shared_file, exception
)
else:
try:
shared_file.delete()
except OperationalError as exception:
logger.warning(
'Operational error during attempt to delete shared '
'file: %s; %s.', shared_file, exception
)
@app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True) @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True)
def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None): def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None):
SharedUploadedFile = apps.get_model( SharedUploadedFile = apps.get_model(

View File

@@ -228,9 +228,6 @@ class DocumentViewTestMixin(object):
data={'id_list': self.test_document.pk} data={'id_list': self.test_document.pk}
) )
def _request_empty_trash_view(self):
return self.post(viewname='documents:trash_can_empty')
def _request_document_print_view(self): def _request_document_print_view(self):
return self.get( return self.get(
viewname='documents:document_print', kwargs={ viewname='documents:document_print', kwargs={
@@ -239,3 +236,52 @@ class DocumentViewTestMixin(object):
'page_group': PAGE_RANGE_ALL 'page_group': PAGE_RANGE_ALL
} }
) )
class DocumentTrashViewMixin(object):
def _request_document_delete_get_view(self):
return self.get(
viewname='documents:document_delete', kwargs={
'pk': self.test_document.pk
}
)
def _request_document_delete_post_view(self):
return self.post(
viewname='documents:document_delete', kwargs={
'pk': self.test_document.pk
}
)
def _request_document_list_deleted_view(self):
return self.get(viewname='documents:document_list_deleted')
def _request_document_restore_get_view(self):
return self.get(
viewname='documents:document_restore', kwargs={
'pk': self.test_document.pk
}
)
def _request_document_restore_post_view(self):
return self.post(
viewname='documents:document_restore', kwargs={
'pk': self.test_document.pk
}
)
def _request_document_trash_get_view(self):
return self.get(
viewname='documents:document_trash', kwargs={
'pk': self.test_document.pk
}
)
def _request_empty_trash_view(self):
return self.post(viewname='documents:trash_can_empty')
def _request_document_trash_post_view(self):
return self.post(
viewname='documents:document_trash', kwargs={
'pk': self.test_document.pk
}
)

View File

@@ -3,12 +3,16 @@ from __future__ import unicode_literals
from actstream.models import Action from actstream.models import Action
from django_downloadview import assert_download_response from django_downloadview import assert_download_response
from ..events import event_document_download, event_document_view from ..events import (
event_document_download, event_document_trashed, event_document_view
)
from ..permissions import ( from ..permissions import (
permission_document_download, permission_document_view permission_document_download, permission_document_trash,
permission_document_view
) )
from .base import GenericDocumentViewTestCase from .base import GenericDocumentViewTestCase
from .mixins import DocumentTrashViewMixin
TEST_DOCUMENT_TYPE_EDITED_LABEL = 'test document type edited label' TEST_DOCUMENT_TYPE_EDITED_LABEL = 'test document type edited label'
@@ -84,3 +88,20 @@ class DocumentEventsTestCase(GenericDocumentViewTestCase):
self.assertEqual(event.actor, self._test_case_user) self.assertEqual(event.actor, self._test_case_user)
self.assertEqual(event.target, self.test_document) self.assertEqual(event.target, self.test_document)
self.assertEqual(event.verb, event_document_view.id) self.assertEqual(event.verb, event_document_view.id)
class DocumentTrashEventsTestCase(DocumentTrashViewMixin, GenericDocumentViewTestCase):
def test_document_trash_event_with_permissions(self):
Action.objects.all().delete()
self.grant_access(
obj=self.test_document, permission=permission_document_trash
)
response = self._request_document_trash_post_view()
self.assertEqual(response.status_code, 302)
event = Action.objects.any(obj=self.test_document).first()
#self.assertEqual(event.actor, self._test_case_user)
self.assertEqual(event.target, self.test_document)
self.assertEqual(event.verb, event_document_trashed.id)

View File

@@ -7,16 +7,10 @@ from ..permissions import (
) )
from .base import GenericDocumentViewTestCase from .base import GenericDocumentViewTestCase
from .mixins import DocumentTrashViewMixin
class TrashedDocumentTestCase(GenericDocumentViewTestCase): class TrashedDocumentTestCase(DocumentTrashViewMixin, GenericDocumentViewTestCase):
def _request_document_restore_get_view(self):
return self.get(
viewname='documents:document_restore', kwargs={
'pk': self.test_document.pk
}
)
def test_document_restore_get_view_no_permission(self): def test_document_restore_get_view_no_permission(self):
self.test_document.delete() self.test_document.delete()
self.assertEqual(Document.objects.count(), 0) self.assertEqual(Document.objects.count(), 0)
@@ -43,13 +37,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
self.assertEqual(Document.objects.count(), document_count) self.assertEqual(Document.objects.count(), document_count)
def _request_document_restore_post_view(self):
return self.post(
viewname='documents:document_restore', kwargs={
'pk': self.test_document.pk
}
)
def test_document_restore_post_view_no_permission(self): def test_document_restore_post_view_no_permission(self):
self.test_document.delete() self.test_document.delete()
self.assertEqual(Document.objects.count(), 0) self.assertEqual(Document.objects.count(), 0)
@@ -74,13 +61,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(DeletedDocument.objects.count(), 0)
self.assertEqual(Document.objects.count(), 1) self.assertEqual(Document.objects.count(), 1)
def _request_document_trash_get_view(self):
return self.get(
viewname='documents:document_trash', kwargs={
'pk': self.test_document.pk
}
)
def test_document_trash_get_view_no_permissions(self): def test_document_trash_get_view_no_permissions(self):
document_count = Document.objects.count() document_count = Document.objects.count()
@@ -101,13 +81,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
self.assertEqual(Document.objects.count(), document_count) self.assertEqual(Document.objects.count(), document_count)
def _request_document_trash_post_view(self):
return self.post(
viewname='documents:document_trash', kwargs={
'pk': self.test_document.pk
}
)
def test_document_trash_post_view_no_permissions(self): def test_document_trash_post_view_no_permissions(self):
response = self._request_document_trash_post_view() response = self._request_document_trash_post_view()
self.assertEqual(response.status_code, 404) self.assertEqual(response.status_code, 404)
@@ -126,13 +99,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
self.assertEqual(DeletedDocument.objects.count(), 1) self.assertEqual(DeletedDocument.objects.count(), 1)
self.assertEqual(Document.objects.count(), 0) self.assertEqual(Document.objects.count(), 0)
def _request_document_delete_get_view(self):
return self.get(
viewname='documents:document_delete', kwargs={
'pk': self.test_document.pk
}
)
def test_document_delete_get_view_no_permissions(self): def test_document_delete_get_view_no_permissions(self):
self.test_document.delete() self.test_document.delete()
self.assertEqual(Document.objects.count(), 0) self.assertEqual(Document.objects.count(), 0)
@@ -165,13 +131,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
DeletedDocument.objects.count(), trashed_document_count DeletedDocument.objects.count(), trashed_document_count
) )
def _request_document_delete_post_view(self):
return self.post(
viewname='documents:document_delete', kwargs={
'pk': self.test_document.pk
}
)
def test_document_delete_post_view_no_permissions(self): def test_document_delete_post_view_no_permissions(self):
self.test_document.delete() self.test_document.delete()
self.assertEqual(Document.objects.count(), 0) self.assertEqual(Document.objects.count(), 0)
@@ -198,9 +157,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(DeletedDocument.objects.count(), 0)
self.assertEqual(Document.objects.count(), 0) self.assertEqual(Document.objects.count(), 0)
def _request_document_list_deleted_view(self):
return self.get(viewname='documents:document_list_deleted')
def test_deleted_document_list_view_no_permissions(self): def test_deleted_document_list_view_no_permissions(self):
self.test_document.delete() self.test_document.delete()

View File

@@ -1,3 +0,0 @@
from __future__ import unicode_literals
default_app_config = 'mayan.apps.importer.apps.ImporterApp'

View File

@@ -1,17 +0,0 @@
from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.apps import MayanAppConfig
class ImporterApp(MayanAppConfig):
app_namespace = 'importer'
app_url = 'importer'
has_rest_api = False
has_tests = True
name = 'mayan.apps.importer'
verbose_name = _('Importer')
def ready(self):
super(ImporterApp, self).ready()

View File

@@ -1,150 +0,0 @@
from __future__ import unicode_literals
import csv
import time
from django.apps import apps
from django.core import management
from django.core.files import File
from ...tasks import task_upload_new_document
class Command(management.BaseCommand):
help = 'Import documents from a CSV file.'
def add_arguments(self, parser):
parser.add_argument(
'--document_type_column',
action='store', dest='document_type_column', default=0,
help='Column that contains the document type labels. Column '
'numbers start at 0.',
type=int
)
parser.add_argument(
'--document_path_column',
action='store', dest='document_path_column', default=1,
help='Column that contains the path to the document files. Column '
'numbers start at 0.',
type=int
)
parser.add_argument(
'--ignore_errors',
action='store_true', dest='ignore_errors', default=False,
help='Don\'t stop the import process on common errors like '
'incorrect file paths.',
)
parser.add_argument(
'--ignore_rows',
action='store', dest='ignore_rows', default='',
help='Ignore a set of rows. Row numbers must be separated by commas.'
)
parser.add_argument(
'--metadata_pairs_column',
action='store', dest='metadata_pairs_column',
help='Column that contains metadata name and values for the '
'documents. Use the form: <label column>:<value column>. Example: '
'2:5. Separate multiple pairs with commas. Example: 2:5,7:10',
)
parser.add_argument('filelist', nargs='?', help='File list')
def handle(self, *args, **options):
time_start = time.time()
time_last_display = time_start
document_types = {}
uploaded_count = 0
row_count = 0
rows_to_ignore = []
for entry in options['ignore_rows'].split(','):
if entry:
rows_to_ignore.append(int(entry))
DocumentType = apps.get_model(
app_label='documents', model_name='DocumentType'
)
SharedUploadedFile = apps.get_model(
app_label='common', model_name='SharedUploadedFile'
)
if not options['filelist']:
self.stderr.write('Must specify a CSV file path.')
exit(1)
else:
with open(options['filelist']) as csv_datafile:
csv_reader = csv.reader(csv_datafile)
for row in csv_reader:
# Increase row count here even though start index is 0
# purpose is to avoid losing row number increments on
# exceptions
row_count = row_count + 1
if row_count - 1 not in rows_to_ignore:
try:
with open(row[options['document_path_column']]) as file_object:
document_type_label = row[options['document_type_column']]
if document_type_label not in document_types:
self.stdout.write(
'New document type: {}. Creating and caching.'.format(
document_type_label
)
)
document_type, created = DocumentType.objects.get_or_create(
label=document_type_label
)
document_types[document_type_label] = document_type
else:
document_type = document_types[document_type_label]
shared_uploaded_file = SharedUploadedFile.objects.create(
file=File(file_object)
)
extra_data = {}
if options['metadata_pairs_column']:
extra_data['metadata_pairs'] = []
for pair in options['metadata_pairs_column'].split(','):
name, value = pair.split(':')
extra_data['metadata_pairs'].append(
{
'name': row[int(name)],
'value': row[int(value)]
}
)
task_upload_new_document.apply_async(
kwargs=dict(
document_type_id=document_type.pk,
shared_uploaded_file_id=shared_uploaded_file.pk,
extra_data=extra_data
)
)
uploaded_count = uploaded_count + 1
if (time.time() - time_last_display) > 1:
time_last_display = time.time()
self.stdout.write(
'Time: {}s, Files copied and queued: {}, files processed per second: {}'.format(
int(time.time() - time_start),
uploaded_count,
uploaded_count / (time.time() - time_start)
)
)
except (IOError, OSError) as exception:
if not options['ignore_errors']:
raise
else:
self.stderr.write(
'Error processing row: {}; {}.'.format(
row_count - 1, exception
)
)
self.stdout.write(
'Total files copied and queues: {}'.format(uploaded_count)
)
self.stdout.write(
'Total time: {}'.format(time.time() - time_start)
)

View File

@@ -1,10 +0,0 @@
from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.documents.queues import queue_uploads
queue_uploads.add_task_type(
dotted_path='mayan.apps.importer.tasks.task_upload_new_document',
label=_('Import new document')
)

View File

@@ -1,92 +0,0 @@
from __future__ import unicode_literals
import logging
from django.apps import apps
from django.db import OperationalError
from django.utils.text import slugify
from mayan.celery import app
from mayan.apps.documents.literals import UPLOAD_NEW_DOCUMENT_RETRY_DELAY
logger = logging.getLogger(__name__)
@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
def task_upload_new_document(self, document_type_id, shared_uploaded_file_id, extra_data=None):
DocumentType = apps.get_model(
app_label='documents', model_name='DocumentType'
)
MetadataType = apps.get_model(
app_label='metadata', model_name='MetadataType'
)
SharedUploadedFile = apps.get_model(
app_label='common', model_name='SharedUploadedFile'
)
try:
document_type = DocumentType.objects.get(pk=document_type_id)
shared_file = SharedUploadedFile.objects.get(
pk=shared_uploaded_file_id
)
except OperationalError as exception:
logger.warning(
'Operational error during attempt to retrieve shared data for '
'new document of type: %s; %s. Retrying.', document_type, exception
)
raise self.retry(exc=exception)
try:
with shared_file.open() as file_object:
new_document = document_type.new_document(file_object=file_object)
except OperationalError as exception:
logger.warning(
'Operational error during attempt to create new document '
'of type: %s; %s. Retrying.', document_type, exception
)
raise self.retry(exc=exception)
except Exception as exception:
# This except and else block emulate a finally:
logger.error(
'Unexpected error during attempt to create new document '
'of type: %s; %s', document_type, exception
)
try:
shared_file.delete()
except OperationalError as exception:
logger.warning(
'Operational error during attempt to delete shared '
'file: %s; %s.', shared_file, exception
)
else:
if extra_data:
for pair in extra_data.get('metadata_pairs', []):
name = slugify(pair['name']).replace('-', '_')
logger.debug(
'Metadata pair (label, name, value): %s, %s, %s',
pair['name'], name, pair['value']
)
metadata_type, created = MetadataType.objects.get_or_create(
label=pair['name'], name=name
)
if not new_document.document_type.metadata.filter(metadata_type=metadata_type).exists():
logger.debug('Metadata type created')
new_document.document_type.metadata.create(
metadata_type=metadata_type, required=False
)
new_document.metadata.create(
metadata_type=metadata_type, value=pair['value']
)
try:
shared_file.delete()
except OperationalError as exception:
logger.warning(
'Operational error during attempt to delete shared '
'file: %s; %s.', shared_file, exception
)

View File

@@ -1,94 +0,0 @@
from __future__ import unicode_literals
import csv
from django.core import management
from django.utils.encoding import force_bytes
from mayan.apps.documents.models import DocumentType, Document
from mayan.apps.documents.tests import GenericDocumentTestCase
from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH
from mayan.apps.storage.utils import fs_cleanup, mkstemp
class ImportManagementCommandTestCase(GenericDocumentTestCase):
auto_upload_document = False
random_primary_key_enable = False
test_import_count = 1
def setUp(self):
super(ImportManagementCommandTestCase, self).setUp()
self._create_test_csv_file()
def tearDown(self):
self._destroy_test_csv_file()
super(ImportManagementCommandTestCase, self).tearDown()
def _create_test_csv_file(self):
self.test_csv_file_descriptor, self.test_csv_path = mkstemp()
print('Test CSV file: {}'.format(self.test_csv_path))
with open(self.test_csv_path, mode='wb') as csvfile:
filewriter = csv.writer(
csvfile, delimiter=force_bytes(','), quotechar=force_bytes('"'),
quoting=csv.QUOTE_MINIMAL
)
print(
'Generating test CSV for {} documents'.format(
self.test_import_count
)
)
for times in range(self.test_import_count):
filewriter.writerow(
[
self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH,
'column 2', 'column 3', 'column 4', 'column 5',
'column 6', 'column 7', 'column 8', 'column 9',
'column 10', 'column 11',
]
)
def _destroy_test_csv_file(self):
fs_cleanup(
filename=self.test_csv_path,
file_descriptor=self.test_csv_file_descriptor
)
def test_import_csv_read(self):
self.test_document_type.delete()
management.call_command('import', self.test_csv_path)
self.assertTrue(DocumentType.objects.count() > 0)
self.assertTrue(Document.objects.count() > 0)
def test_import_document_type_column_mapping(self):
self.test_document_type.delete()
management.call_command(
'import', self.test_csv_path, '--document_type_column', '2'
)
self.assertTrue(DocumentType.objects.first().label == 'column 2')
self.assertTrue(Document.objects.count() > 0)
def test_import_document_path_column_mapping(self):
self.test_document_type.delete()
with self.assertRaises(IOError):
management.call_command(
'import', self.test_csv_path, '--document_path_column', '2'
)
def test_import_metadata_column_mapping(self):
self.test_document_type.delete()
management.call_command(
'import', self.test_csv_path, '--metadata_pairs_column', '2:3,4:5',
)
self.assertTrue(DocumentType.objects.count() > 0)
self.assertTrue(Document.objects.count() > 0)
self.assertTrue(Document.objects.first().metadata.count() > 0)
self.assertEqual(
Document.objects.first().metadata.get(
metadata_type__name='column_2'
).value, 'column 3'
)

View File

@@ -16,7 +16,8 @@ class Command(management.BaseCommand):
) )
parser.add_argument( parser.add_argument(
'--context', action='store', default='', dest='context', '--context', action='store', default='', dest='context',
help='Show a list of available templates.', help='Pass a context to the template in the form of a JSON encoded '
'dictionary.',
) )
def handle(self, *args, **options): def handle(self, *args, **options):

View File

@@ -120,7 +120,6 @@ INSTALLED_APPS = (
'mayan.apps.document_states', 'mayan.apps.document_states',
'mayan.apps.documents', 'mayan.apps.documents',
'mayan.apps.file_metadata', 'mayan.apps.file_metadata',
'mayan.apps.importer',
'mayan.apps.linking', 'mayan.apps.linking',
'mayan.apps.mailer', 'mayan.apps.mailer',
'mayan.apps.mayan_statistics', 'mayan.apps.mayan_statistics',