Compare commits
2 Commits
nightly
...
features/t
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
adf47646bf | ||
|
|
4d7c0552bd |
10
HISTORY.rst
10
HISTORY.rst
@@ -1,19 +1,13 @@
|
||||
Importer branch
|
||||
===============
|
||||
* Add a reusable task to upload documents.
|
||||
* Add MVP of the importer app.
|
||||
|
||||
3.2.4 (2019-06-XX)
|
||||
==================
|
||||
* Support configurable GUnicorn timeouts. Defaults to
|
||||
* Support configurable GUnicorn timeouts. Defaults to
|
||||
current value of 120 seconds.
|
||||
* Fix help text of the platformtemplate command.
|
||||
|
||||
3.2.3 (2019-06-21)
|
||||
==================
|
||||
* Add support for disabling the random primary key
|
||||
test mixin.
|
||||
* Add a reusable task to upload documents.
|
||||
* Add MVP of the importer app.
|
||||
* Fix mailing profile log columns mappings.
|
||||
GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
|
||||
for the report.
|
||||
|
||||
@@ -19,6 +19,7 @@ Changes
|
||||
GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
|
||||
for the report and the research.
|
||||
|
||||
|
||||
Removals
|
||||
--------
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ Changes
|
||||
|
||||
- Support configurable GUnicorn timeouts. Defaults to
|
||||
current value of 120 seconds.
|
||||
|
||||
- Fix help text of the platformtemplate command.
|
||||
|
||||
Removals
|
||||
--------
|
||||
|
||||
@@ -18,6 +18,9 @@ event_document_new_version = namespace.add_event_type(
|
||||
event_document_properties_edit = namespace.add_event_type(
|
||||
label=_('Document properties edited'), name='document_edit'
|
||||
)
|
||||
event_document_trashed = namespace.add_event_type(
|
||||
label=_('Document moved to trash'), name='document_trashed'
|
||||
)
|
||||
# The type of an existing document is changed to another type
|
||||
event_document_type_change = namespace.add_event_type(
|
||||
label=_('Document type changed'), name='document_type_change'
|
||||
|
||||
@@ -32,7 +32,6 @@ DEFAULT_DOCUMENT_TYPE_LABEL = _('Default')
|
||||
DOCUMENT_IMAGE_TASK_TIMEOUT = 120
|
||||
STUB_EXPIRATION_INTERVAL = 60 * 60 * 24 # 24 hours
|
||||
UPDATE_PAGE_COUNT_RETRY_DELAY = 10
|
||||
UPLOAD_NEW_DOCUMENT_RETRY_DELAY = 10
|
||||
UPLOAD_NEW_VERSION_RETRY_DELAY = 10
|
||||
|
||||
PAGE_RANGE_ALL = 'all'
|
||||
|
||||
@@ -5,7 +5,7 @@ import uuid
|
||||
|
||||
from django.apps import apps
|
||||
from django.core.files import File
|
||||
from django.db import models
|
||||
from django.db import models, transaction
|
||||
from django.urls import reverse
|
||||
from django.utils.encoding import python_2_unicode_compatible
|
||||
from django.utils.timezone import now
|
||||
@@ -13,7 +13,7 @@ from django.utils.translation import ugettext, ugettext_lazy as _
|
||||
|
||||
from ..events import (
|
||||
event_document_create, event_document_properties_edit,
|
||||
event_document_type_change,
|
||||
event_document_trashed, event_document_type_change,
|
||||
)
|
||||
from ..managers import DocumentManager, PassthroughManager, TrashCanManager
|
||||
from ..settings import setting_language
|
||||
@@ -104,11 +104,14 @@ class Document(models.Model):
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
to_trash = kwargs.pop('to_trash', True)
|
||||
_user = kwargs.pop('_user', None)
|
||||
|
||||
if not self.in_trash and to_trash:
|
||||
self.in_trash = True
|
||||
self.deleted_date_time = now()
|
||||
self.save()
|
||||
with transaction.atomic():
|
||||
self.in_trash = True
|
||||
self.deleted_date_time = now()
|
||||
self.save()
|
||||
event_document_trashed.commit(actor=_user, target=self)
|
||||
else:
|
||||
for version in self.versions.all():
|
||||
version.delete()
|
||||
@@ -177,21 +180,23 @@ class Document(models.Model):
|
||||
user = kwargs.pop('_user', None)
|
||||
_commit_events = kwargs.pop('_commit_events', True)
|
||||
new_document = not self.pk
|
||||
super(Document, self).save(*args, **kwargs)
|
||||
|
||||
if new_document:
|
||||
if user:
|
||||
self.add_as_recent_document_for_user(user)
|
||||
event_document_create.commit(
|
||||
actor=user, target=self, action_object=self.document_type
|
||||
)
|
||||
with transaction.atomic():
|
||||
super(Document, self).save(*args, **kwargs)
|
||||
|
||||
if new_document:
|
||||
if user:
|
||||
self.add_as_recent_document_for_user(user)
|
||||
event_document_create.commit(
|
||||
actor=user, target=self, action_object=self.document_type
|
||||
)
|
||||
else:
|
||||
event_document_create.commit(
|
||||
target=self, action_object=self.document_type
|
||||
)
|
||||
else:
|
||||
event_document_create.commit(
|
||||
target=self, action_object=self.document_type
|
||||
)
|
||||
else:
|
||||
if _commit_events:
|
||||
event_document_properties_edit.commit(actor=user, target=self)
|
||||
if _commit_events:
|
||||
event_document_properties_edit.commit(actor=user, target=self)
|
||||
|
||||
def save_to_file(self, *args, **kwargs):
|
||||
return self.latest_version.save_to_file(*args, **kwargs)
|
||||
@@ -200,15 +205,16 @@ class Document(models.Model):
|
||||
has_changed = self.document_type != document_type
|
||||
|
||||
self.document_type = document_type
|
||||
self.save()
|
||||
if has_changed or force:
|
||||
post_document_type_change.send(
|
||||
sender=self.__class__, instance=self
|
||||
)
|
||||
with transaction.atomic():
|
||||
self.save()
|
||||
if has_changed or force:
|
||||
post_document_type_change.send(
|
||||
sender=self.__class__, instance=self
|
||||
)
|
||||
|
||||
event_document_type_change.commit(actor=_user, target=self)
|
||||
if _user:
|
||||
self.add_as_recent_document_for_user(user=_user)
|
||||
event_document_type_change.commit(actor=_user, target=self)
|
||||
if _user:
|
||||
self.add_as_recent_document_for_user(user=_user)
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
|
||||
@@ -82,7 +82,3 @@ queue_uploads.add_task_type(
|
||||
dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for',
|
||||
label=_('Scan document duplicates')
|
||||
)
|
||||
queue_uploads.add_task_type(
|
||||
dotted_path='mayan.apps.documents.tasks.task_upload_new_document',
|
||||
label=_('Upload new document')
|
||||
)
|
||||
|
||||
@@ -9,8 +9,7 @@ from django.db import OperationalError
|
||||
from mayan.celery import app
|
||||
|
||||
from .literals import (
|
||||
UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY,
|
||||
UPLOAD_NEW_VERSION_RETRY_DELAY
|
||||
UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -128,60 +127,6 @@ def task_update_page_count(self, version_id):
|
||||
raise self.retry(exc=exception)
|
||||
|
||||
|
||||
@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
|
||||
def task_upload_new_document(self, document_type_id, shared_uploaded_file_id):
|
||||
DocumentType = apps.get_model(
|
||||
app_label='documents', model_name='DocumentType'
|
||||
)
|
||||
|
||||
SharedUploadedFile = apps.get_model(
|
||||
app_label='common', model_name='SharedUploadedFile'
|
||||
)
|
||||
|
||||
try:
|
||||
document_type = DocumentType.objects.get(pk=document_type_id)
|
||||
shared_file = SharedUploadedFile.objects.get(
|
||||
pk=shared_uploaded_file_id
|
||||
)
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to retrieve shared data for '
|
||||
'new document of type: %s; %s. Retrying.', document_type, exception
|
||||
)
|
||||
raise self.retry(exc=exception)
|
||||
|
||||
try:
|
||||
with shared_file.open() as file_object:
|
||||
document_type.new_document(file_object=file_object)
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to create new document '
|
||||
'of type: %s; %s. Retrying.', document_type, exception
|
||||
)
|
||||
raise self.retry(exc=exception)
|
||||
except Exception as exception:
|
||||
# This except and else block emulate a finally:
|
||||
logger.error(
|
||||
'Unexpected error during attempt to create new document '
|
||||
'of type: %s; %s', document_type, exception
|
||||
)
|
||||
try:
|
||||
shared_file.delete()
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to delete shared '
|
||||
'file: %s; %s.', shared_file, exception
|
||||
)
|
||||
else:
|
||||
try:
|
||||
shared_file.delete()
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to delete shared '
|
||||
'file: %s; %s.', shared_file, exception
|
||||
)
|
||||
|
||||
|
||||
@app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True)
|
||||
def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None):
|
||||
SharedUploadedFile = apps.get_model(
|
||||
|
||||
@@ -228,9 +228,6 @@ class DocumentViewTestMixin(object):
|
||||
data={'id_list': self.test_document.pk}
|
||||
)
|
||||
|
||||
def _request_empty_trash_view(self):
|
||||
return self.post(viewname='documents:trash_can_empty')
|
||||
|
||||
def _request_document_print_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_print', kwargs={
|
||||
@@ -239,3 +236,52 @@ class DocumentViewTestMixin(object):
|
||||
'page_group': PAGE_RANGE_ALL
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class DocumentTrashViewMixin(object):
|
||||
def _request_document_delete_get_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_delete', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def _request_document_delete_post_view(self):
|
||||
return self.post(
|
||||
viewname='documents:document_delete', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def _request_document_list_deleted_view(self):
|
||||
return self.get(viewname='documents:document_list_deleted')
|
||||
|
||||
def _request_document_restore_get_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_restore', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
def _request_document_restore_post_view(self):
|
||||
return self.post(
|
||||
viewname='documents:document_restore', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def _request_document_trash_get_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_trash', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def _request_empty_trash_view(self):
|
||||
return self.post(viewname='documents:trash_can_empty')
|
||||
|
||||
def _request_document_trash_post_view(self):
|
||||
return self.post(
|
||||
viewname='documents:document_trash', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3,12 +3,16 @@ from __future__ import unicode_literals
|
||||
from actstream.models import Action
|
||||
from django_downloadview import assert_download_response
|
||||
|
||||
from ..events import event_document_download, event_document_view
|
||||
from ..events import (
|
||||
event_document_download, event_document_trashed, event_document_view
|
||||
)
|
||||
from ..permissions import (
|
||||
permission_document_download, permission_document_view
|
||||
permission_document_download, permission_document_trash,
|
||||
permission_document_view
|
||||
)
|
||||
|
||||
from .base import GenericDocumentViewTestCase
|
||||
from .mixins import DocumentTrashViewMixin
|
||||
|
||||
|
||||
TEST_DOCUMENT_TYPE_EDITED_LABEL = 'test document type edited label'
|
||||
@@ -84,3 +88,20 @@ class DocumentEventsTestCase(GenericDocumentViewTestCase):
|
||||
self.assertEqual(event.actor, self._test_case_user)
|
||||
self.assertEqual(event.target, self.test_document)
|
||||
self.assertEqual(event.verb, event_document_view.id)
|
||||
|
||||
|
||||
class DocumentTrashEventsTestCase(DocumentTrashViewMixin, GenericDocumentViewTestCase):
|
||||
def test_document_trash_event_with_permissions(self):
|
||||
Action.objects.all().delete()
|
||||
|
||||
self.grant_access(
|
||||
obj=self.test_document, permission=permission_document_trash
|
||||
)
|
||||
|
||||
response = self._request_document_trash_post_view()
|
||||
self.assertEqual(response.status_code, 302)
|
||||
|
||||
event = Action.objects.any(obj=self.test_document).first()
|
||||
#self.assertEqual(event.actor, self._test_case_user)
|
||||
self.assertEqual(event.target, self.test_document)
|
||||
self.assertEqual(event.verb, event_document_trashed.id)
|
||||
|
||||
@@ -7,16 +7,10 @@ from ..permissions import (
|
||||
)
|
||||
|
||||
from .base import GenericDocumentViewTestCase
|
||||
from .mixins import DocumentTrashViewMixin
|
||||
|
||||
|
||||
class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
def _request_document_restore_get_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_restore', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
class TrashedDocumentTestCase(DocumentTrashViewMixin, GenericDocumentViewTestCase):
|
||||
def test_document_restore_get_view_no_permission(self):
|
||||
self.test_document.delete()
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
@@ -43,13 +37,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
|
||||
self.assertEqual(Document.objects.count(), document_count)
|
||||
|
||||
def _request_document_restore_post_view(self):
|
||||
return self.post(
|
||||
viewname='documents:document_restore', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def test_document_restore_post_view_no_permission(self):
|
||||
self.test_document.delete()
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
@@ -74,13 +61,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
self.assertEqual(DeletedDocument.objects.count(), 0)
|
||||
self.assertEqual(Document.objects.count(), 1)
|
||||
|
||||
def _request_document_trash_get_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_trash', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def test_document_trash_get_view_no_permissions(self):
|
||||
document_count = Document.objects.count()
|
||||
|
||||
@@ -101,13 +81,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
|
||||
self.assertEqual(Document.objects.count(), document_count)
|
||||
|
||||
def _request_document_trash_post_view(self):
|
||||
return self.post(
|
||||
viewname='documents:document_trash', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def test_document_trash_post_view_no_permissions(self):
|
||||
response = self._request_document_trash_post_view()
|
||||
self.assertEqual(response.status_code, 404)
|
||||
@@ -126,13 +99,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
self.assertEqual(DeletedDocument.objects.count(), 1)
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
|
||||
def _request_document_delete_get_view(self):
|
||||
return self.get(
|
||||
viewname='documents:document_delete', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def test_document_delete_get_view_no_permissions(self):
|
||||
self.test_document.delete()
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
@@ -165,13 +131,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
DeletedDocument.objects.count(), trashed_document_count
|
||||
)
|
||||
|
||||
def _request_document_delete_post_view(self):
|
||||
return self.post(
|
||||
viewname='documents:document_delete', kwargs={
|
||||
'pk': self.test_document.pk
|
||||
}
|
||||
)
|
||||
|
||||
def test_document_delete_post_view_no_permissions(self):
|
||||
self.test_document.delete()
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
@@ -198,9 +157,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase):
|
||||
self.assertEqual(DeletedDocument.objects.count(), 0)
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
|
||||
def _request_document_list_deleted_view(self):
|
||||
return self.get(viewname='documents:document_list_deleted')
|
||||
|
||||
def test_deleted_document_list_view_no_permissions(self):
|
||||
self.test_document.delete()
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
default_app_config = 'mayan.apps.importer.apps.ImporterApp'
|
||||
@@ -1,17 +0,0 @@
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from mayan.apps.common.apps import MayanAppConfig
|
||||
|
||||
|
||||
class ImporterApp(MayanAppConfig):
|
||||
app_namespace = 'importer'
|
||||
app_url = 'importer'
|
||||
has_rest_api = False
|
||||
has_tests = True
|
||||
name = 'mayan.apps.importer'
|
||||
verbose_name = _('Importer')
|
||||
|
||||
def ready(self):
|
||||
super(ImporterApp, self).ready()
|
||||
@@ -1,150 +0,0 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import csv
|
||||
import time
|
||||
|
||||
from django.apps import apps
|
||||
from django.core import management
|
||||
from django.core.files import File
|
||||
|
||||
from ...tasks import task_upload_new_document
|
||||
|
||||
|
||||
class Command(management.BaseCommand):
|
||||
help = 'Import documents from a CSV file.'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'--document_type_column',
|
||||
action='store', dest='document_type_column', default=0,
|
||||
help='Column that contains the document type labels. Column '
|
||||
'numbers start at 0.',
|
||||
type=int
|
||||
)
|
||||
parser.add_argument(
|
||||
'--document_path_column',
|
||||
action='store', dest='document_path_column', default=1,
|
||||
help='Column that contains the path to the document files. Column '
|
||||
'numbers start at 0.',
|
||||
type=int
|
||||
)
|
||||
parser.add_argument(
|
||||
'--ignore_errors',
|
||||
action='store_true', dest='ignore_errors', default=False,
|
||||
help='Don\'t stop the import process on common errors like '
|
||||
'incorrect file paths.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--ignore_rows',
|
||||
action='store', dest='ignore_rows', default='',
|
||||
help='Ignore a set of rows. Row numbers must be separated by commas.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--metadata_pairs_column',
|
||||
action='store', dest='metadata_pairs_column',
|
||||
help='Column that contains metadata name and values for the '
|
||||
'documents. Use the form: <label column>:<value column>. Example: '
|
||||
'2:5. Separate multiple pairs with commas. Example: 2:5,7:10',
|
||||
)
|
||||
parser.add_argument('filelist', nargs='?', help='File list')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
time_start = time.time()
|
||||
time_last_display = time_start
|
||||
document_types = {}
|
||||
uploaded_count = 0
|
||||
row_count = 0
|
||||
rows_to_ignore = []
|
||||
for entry in options['ignore_rows'].split(','):
|
||||
if entry:
|
||||
rows_to_ignore.append(int(entry))
|
||||
|
||||
DocumentType = apps.get_model(
|
||||
app_label='documents', model_name='DocumentType'
|
||||
)
|
||||
SharedUploadedFile = apps.get_model(
|
||||
app_label='common', model_name='SharedUploadedFile'
|
||||
)
|
||||
|
||||
if not options['filelist']:
|
||||
self.stderr.write('Must specify a CSV file path.')
|
||||
exit(1)
|
||||
else:
|
||||
with open(options['filelist']) as csv_datafile:
|
||||
csv_reader = csv.reader(csv_datafile)
|
||||
for row in csv_reader:
|
||||
# Increase row count here even though start index is 0
|
||||
# purpose is to avoid losing row number increments on
|
||||
# exceptions
|
||||
row_count = row_count + 1
|
||||
if row_count - 1 not in rows_to_ignore:
|
||||
try:
|
||||
with open(row[options['document_path_column']]) as file_object:
|
||||
document_type_label = row[options['document_type_column']]
|
||||
|
||||
if document_type_label not in document_types:
|
||||
self.stdout.write(
|
||||
'New document type: {}. Creating and caching.'.format(
|
||||
document_type_label
|
||||
)
|
||||
)
|
||||
document_type, created = DocumentType.objects.get_or_create(
|
||||
label=document_type_label
|
||||
)
|
||||
document_types[document_type_label] = document_type
|
||||
else:
|
||||
document_type = document_types[document_type_label]
|
||||
|
||||
shared_uploaded_file = SharedUploadedFile.objects.create(
|
||||
file=File(file_object)
|
||||
)
|
||||
|
||||
extra_data = {}
|
||||
if options['metadata_pairs_column']:
|
||||
extra_data['metadata_pairs'] = []
|
||||
|
||||
for pair in options['metadata_pairs_column'].split(','):
|
||||
name, value = pair.split(':')
|
||||
extra_data['metadata_pairs'].append(
|
||||
{
|
||||
'name': row[int(name)],
|
||||
'value': row[int(value)]
|
||||
}
|
||||
)
|
||||
|
||||
task_upload_new_document.apply_async(
|
||||
kwargs=dict(
|
||||
document_type_id=document_type.pk,
|
||||
shared_uploaded_file_id=shared_uploaded_file.pk,
|
||||
extra_data=extra_data
|
||||
)
|
||||
)
|
||||
|
||||
uploaded_count = uploaded_count + 1
|
||||
|
||||
if (time.time() - time_last_display) > 1:
|
||||
time_last_display = time.time()
|
||||
self.stdout.write(
|
||||
'Time: {}s, Files copied and queued: {}, files processed per second: {}'.format(
|
||||
int(time.time() - time_start),
|
||||
uploaded_count,
|
||||
uploaded_count / (time.time() - time_start)
|
||||
)
|
||||
)
|
||||
|
||||
except (IOError, OSError) as exception:
|
||||
if not options['ignore_errors']:
|
||||
raise
|
||||
else:
|
||||
self.stderr.write(
|
||||
'Error processing row: {}; {}.'.format(
|
||||
row_count - 1, exception
|
||||
)
|
||||
)
|
||||
|
||||
self.stdout.write(
|
||||
'Total files copied and queues: {}'.format(uploaded_count)
|
||||
)
|
||||
self.stdout.write(
|
||||
'Total time: {}'.format(time.time() - time_start)
|
||||
)
|
||||
@@ -1,10 +0,0 @@
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from mayan.apps.documents.queues import queue_uploads
|
||||
|
||||
queue_uploads.add_task_type(
|
||||
dotted_path='mayan.apps.importer.tasks.task_upload_new_document',
|
||||
label=_('Import new document')
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
|
||||
from django.apps import apps
|
||||
from django.db import OperationalError
|
||||
from django.utils.text import slugify
|
||||
|
||||
from mayan.celery import app
|
||||
|
||||
from mayan.apps.documents.literals import UPLOAD_NEW_DOCUMENT_RETRY_DELAY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
|
||||
def task_upload_new_document(self, document_type_id, shared_uploaded_file_id, extra_data=None):
|
||||
DocumentType = apps.get_model(
|
||||
app_label='documents', model_name='DocumentType'
|
||||
)
|
||||
|
||||
MetadataType = apps.get_model(
|
||||
app_label='metadata', model_name='MetadataType'
|
||||
)
|
||||
|
||||
SharedUploadedFile = apps.get_model(
|
||||
app_label='common', model_name='SharedUploadedFile'
|
||||
)
|
||||
|
||||
try:
|
||||
document_type = DocumentType.objects.get(pk=document_type_id)
|
||||
shared_file = SharedUploadedFile.objects.get(
|
||||
pk=shared_uploaded_file_id
|
||||
)
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to retrieve shared data for '
|
||||
'new document of type: %s; %s. Retrying.', document_type, exception
|
||||
)
|
||||
raise self.retry(exc=exception)
|
||||
|
||||
try:
|
||||
with shared_file.open() as file_object:
|
||||
new_document = document_type.new_document(file_object=file_object)
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to create new document '
|
||||
'of type: %s; %s. Retrying.', document_type, exception
|
||||
)
|
||||
raise self.retry(exc=exception)
|
||||
except Exception as exception:
|
||||
# This except and else block emulate a finally:
|
||||
logger.error(
|
||||
'Unexpected error during attempt to create new document '
|
||||
'of type: %s; %s', document_type, exception
|
||||
)
|
||||
try:
|
||||
shared_file.delete()
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to delete shared '
|
||||
'file: %s; %s.', shared_file, exception
|
||||
)
|
||||
else:
|
||||
if extra_data:
|
||||
for pair in extra_data.get('metadata_pairs', []):
|
||||
name = slugify(pair['name']).replace('-', '_')
|
||||
logger.debug(
|
||||
'Metadata pair (label, name, value): %s, %s, %s',
|
||||
pair['name'], name, pair['value']
|
||||
)
|
||||
|
||||
metadata_type, created = MetadataType.objects.get_or_create(
|
||||
label=pair['name'], name=name
|
||||
)
|
||||
if not new_document.document_type.metadata.filter(metadata_type=metadata_type).exists():
|
||||
logger.debug('Metadata type created')
|
||||
new_document.document_type.metadata.create(
|
||||
metadata_type=metadata_type, required=False
|
||||
)
|
||||
|
||||
new_document.metadata.create(
|
||||
metadata_type=metadata_type, value=pair['value']
|
||||
)
|
||||
|
||||
try:
|
||||
shared_file.delete()
|
||||
except OperationalError as exception:
|
||||
logger.warning(
|
||||
'Operational error during attempt to delete shared '
|
||||
'file: %s; %s.', shared_file, exception
|
||||
)
|
||||
@@ -1,94 +0,0 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import csv
|
||||
|
||||
from django.core import management
|
||||
from django.utils.encoding import force_bytes
|
||||
|
||||
from mayan.apps.documents.models import DocumentType, Document
|
||||
from mayan.apps.documents.tests import GenericDocumentTestCase
|
||||
from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH
|
||||
from mayan.apps.storage.utils import fs_cleanup, mkstemp
|
||||
|
||||
|
||||
class ImportManagementCommandTestCase(GenericDocumentTestCase):
|
||||
auto_upload_document = False
|
||||
random_primary_key_enable = False
|
||||
test_import_count = 1
|
||||
|
||||
def setUp(self):
|
||||
super(ImportManagementCommandTestCase, self).setUp()
|
||||
self._create_test_csv_file()
|
||||
|
||||
def tearDown(self):
|
||||
self._destroy_test_csv_file()
|
||||
super(ImportManagementCommandTestCase, self).tearDown()
|
||||
|
||||
def _create_test_csv_file(self):
|
||||
self.test_csv_file_descriptor, self.test_csv_path = mkstemp()
|
||||
|
||||
print('Test CSV file: {}'.format(self.test_csv_path))
|
||||
|
||||
with open(self.test_csv_path, mode='wb') as csvfile:
|
||||
filewriter = csv.writer(
|
||||
csvfile, delimiter=force_bytes(','), quotechar=force_bytes('"'),
|
||||
quoting=csv.QUOTE_MINIMAL
|
||||
)
|
||||
print(
|
||||
'Generating test CSV for {} documents'.format(
|
||||
self.test_import_count
|
||||
)
|
||||
)
|
||||
for times in range(self.test_import_count):
|
||||
filewriter.writerow(
|
||||
[
|
||||
self.test_document_type.label, TEST_SMALL_DOCUMENT_PATH,
|
||||
'column 2', 'column 3', 'column 4', 'column 5',
|
||||
'column 6', 'column 7', 'column 8', 'column 9',
|
||||
'column 10', 'column 11',
|
||||
]
|
||||
)
|
||||
|
||||
def _destroy_test_csv_file(self):
|
||||
fs_cleanup(
|
||||
filename=self.test_csv_path,
|
||||
file_descriptor=self.test_csv_file_descriptor
|
||||
)
|
||||
|
||||
def test_import_csv_read(self):
|
||||
self.test_document_type.delete()
|
||||
management.call_command('import', self.test_csv_path)
|
||||
|
||||
self.assertTrue(DocumentType.objects.count() > 0)
|
||||
self.assertTrue(Document.objects.count() > 0)
|
||||
|
||||
def test_import_document_type_column_mapping(self):
|
||||
self.test_document_type.delete()
|
||||
management.call_command(
|
||||
'import', self.test_csv_path, '--document_type_column', '2'
|
||||
)
|
||||
|
||||
self.assertTrue(DocumentType.objects.first().label == 'column 2')
|
||||
self.assertTrue(Document.objects.count() > 0)
|
||||
|
||||
def test_import_document_path_column_mapping(self):
|
||||
self.test_document_type.delete()
|
||||
with self.assertRaises(IOError):
|
||||
management.call_command(
|
||||
'import', self.test_csv_path, '--document_path_column', '2'
|
||||
)
|
||||
|
||||
def test_import_metadata_column_mapping(self):
|
||||
self.test_document_type.delete()
|
||||
management.call_command(
|
||||
'import', self.test_csv_path, '--metadata_pairs_column', '2:3,4:5',
|
||||
)
|
||||
|
||||
self.assertTrue(DocumentType.objects.count() > 0)
|
||||
self.assertTrue(Document.objects.count() > 0)
|
||||
self.assertTrue(Document.objects.first().metadata.count() > 0)
|
||||
self.assertEqual(
|
||||
Document.objects.first().metadata.get(
|
||||
metadata_type__name='column_2'
|
||||
).value, 'column 3'
|
||||
)
|
||||
@@ -16,7 +16,8 @@ class Command(management.BaseCommand):
|
||||
)
|
||||
parser.add_argument(
|
||||
'--context', action='store', default='', dest='context',
|
||||
help='Show a list of available templates.',
|
||||
help='Pass a context to the template in the form of a JSON encoded '
|
||||
'dictionary.',
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
||||
@@ -120,7 +120,6 @@ INSTALLED_APPS = (
|
||||
'mayan.apps.document_states',
|
||||
'mayan.apps.documents',
|
||||
'mayan.apps.file_metadata',
|
||||
'mayan.apps.importer',
|
||||
'mayan.apps.linking',
|
||||
'mayan.apps.mailer',
|
||||
'mayan.apps.mayan_statistics',
|
||||
|
||||
Reference in New Issue
Block a user