diff --git a/mayan/apps/file_metadata/apps.py b/mayan/apps/file_metadata/apps.py index e05cf5c2c9..5e99fa54ec 100644 --- a/mayan/apps/file_metadata/apps.py +++ b/mayan/apps/file_metadata/apps.py @@ -20,6 +20,7 @@ from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn from mayan.celery import app +from .classes import FileMetadataHelper from .drivers import * # NOQA from .events import ( event_file_metadata_document_version_finish, @@ -49,7 +50,7 @@ from .signals import post_document_version_file_metadata_processing class FileMetadataApp(MayanAppConfig): app_namespace = 'file_metadata' app_url = 'file_metadata' - has_test = True + has_tests = True name = 'mayan.apps.file_metadata' verbose_name = _('File metadata') @@ -74,13 +75,16 @@ class FileMetadataApp(MayanAppConfig): ) Document.add_to_class( - name='submit_for_file_metadata_processing', - value=method_document_submit + name='file_metadata_value_of', value=FileMetadataHelper.constructor ) Document.add_to_class( name='get_file_metadata', value=method_get_document_file_metadata ) + Document.add_to_class( + name='submit_for_file_metadata_processing', + value=method_document_submit + ) DocumentVersion.add_to_class( name='get_file_metadata', value=method_get_document_version_file_metadata @@ -90,7 +94,12 @@ class FileMetadataApp(MayanAppConfig): value=method_document_version_submit ) - ModelAttribute(model=Document, name='get_file_metadata') + ModelAttribute( + model=Document, name='file_metadata_value_of', + description=_( + 'Return the value of a specific file metadata.' + ) + ) ModelEventType.register( model=Document, event_types=( @@ -116,7 +125,8 @@ class FileMetadataApp(MayanAppConfig): ModelPermission.register( model=DocumentType, permissions=( permission_document_type_file_metadata_setup, - permission_file_metadata_submit + permission_file_metadata_submit, + permission_file_metadata_view ) ) ModelPermission.register_inheritance( diff --git a/mayan/apps/file_metadata/classes.py b/mayan/apps/file_metadata/classes.py index e1a789c812..6af9b7a79d 100644 --- a/mayan/apps/file_metadata/classes.py +++ b/mayan/apps/file_metadata/classes.py @@ -5,6 +5,8 @@ import logging from django.apps import apps from django.db import transaction +from mayan.apps.common.classes import PropertyHelper + from .events import event_file_metadata_document_version_finish from .exceptions import FileMetadataDriverError from .signals import post_document_version_file_metadata_processing @@ -12,6 +14,18 @@ from .signals import post_document_version_file_metadata_processing logger = logging.getLogger(__name__) +class FileMetadataHelper(PropertyHelper): + @staticmethod + @property + def constructor(*args, **kwargs): + return FileMetadataHelper(*args, **kwargs) + + def get_result(self, name): + name = name.replace('_', '.') + result = self.instance.get_file_metadata(dotted_name=name) + return result + + class FileMetadataDriver(object): _registry = {} diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 6c83f20ee4..ca4295099b 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,11 +4,18 @@ import json import logging import sh +import yaml + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ from mayan.apps.storage.utils import fs_cleanup, mkstemp +from ..literals import DEFAULT_EXIF_PATH from ..classes import FileMetadataDriver from ..settings import setting_drivers_arguments @@ -20,10 +27,16 @@ class EXIFToolDriver(FileMetadataDriver): internal_name = 'exiftool' def __init__(self, *args, **kwargs): + driver_arguments = yaml.load( + stream=setting_drivers_arguments.value, Loader=SafeLoader + ) + + exiftool_path = driver_arguments.get( + 'exif_driver', {} + ).get('exiftool_path', DEFAULT_EXIF_PATH) + try: - self.command_exiftool = sh.Command( - setting_drivers_arguments.value['exif_driver']['exiftool_path'] - ) + self.command_exiftool = sh.Command(path=exiftool_path) except sh.CommandNotFound: self.command_exiftool = None else: @@ -36,13 +49,14 @@ class EXIFToolDriver(FileMetadataDriver): try: document_version.save_to_file(filepath=temp_filename) result = self.command_exiftool(temp_filename) + return json.loads(s=result.stdout)[0] finally: fs_cleanup(filename=temp_filename) else: logger.warning( - 'EXIFTool binary not found, not processing document version: %s', - document_version + 'EXIFTool binary not found, not processing document ' + 'version: %s', document_version ) diff --git a/mayan/apps/file_metadata/models.py b/mayan/apps/file_metadata/models.py index 5d07b194df..585f50abb3 100644 --- a/mayan/apps/file_metadata/models.py +++ b/mayan/apps/file_metadata/models.py @@ -42,31 +42,6 @@ class StoredDriver(models.Model): return self.driver_class.label -@python_2_unicode_compatible -class DocumentVersionDriverEntry(models.Model): - driver = models.ForeignKey( - related_name='driver_entries', to=StoredDriver, - verbose_name=_('Driver') - ) - document_version = models.ForeignKey( - related_name='file_metadata_drivers', to=DocumentVersion, - verbose_name=_('Document version') - ) - - class Meta: - ordering = ('document_version', 'driver') - unique_together = ('driver', 'document_version') - verbose_name = _('Document version driver entry') - verbose_name_plural = _('Document version driver entries') - - def __str__(self): - return force_text(self.driver) - - def get_attribute_count(self): - return self.entries.count() - get_attribute_count.short_description = _('Attribute count') - - class DocumentTypeSettings(models.Model): """ Model to store the file metadata settings for a document type. @@ -92,6 +67,31 @@ class DocumentTypeSettings(models.Model): natural_key.dependencies = ['documents.DocumentType'] +@python_2_unicode_compatible +class DocumentVersionDriverEntry(models.Model): + driver = models.ForeignKey( + related_name='driver_entries', to=StoredDriver, + verbose_name=_('Driver') + ) + document_version = models.ForeignKey( + related_name='file_metadata_drivers', to=DocumentVersion, + verbose_name=_('Document version') + ) + + class Meta: + ordering = ('document_version', 'driver') + unique_together = ('driver', 'document_version') + verbose_name = _('Document version driver entry') + verbose_name_plural = _('Document version driver entries') + + def __str__(self): + return force_text(self.driver) + + def get_attribute_count(self): + return self.entries.count() + get_attribute_count.short_description = _('Attribute count') + + @python_2_unicode_compatible class FileMetadataEntry(models.Model): document_version_driver_entry = models.ForeignKey( diff --git a/mayan/apps/file_metadata/settings.py b/mayan/apps/file_metadata/settings.py index ceb98a3928..247f0b72ec 100644 --- a/mayan/apps/file_metadata/settings.py +++ b/mayan/apps/file_metadata/settings.py @@ -8,12 +8,6 @@ from .literals import DEFAULT_EXIF_PATH namespace = Namespace(label=_('File metadata'), name='file_metadata') -setting_drivers_arguments = namespace.add_setting( - global_name='FILE_METADATA_DRIVERS_ARGUMENTS', - default={'exif_driver': {'exiftool_path': DEFAULT_EXIF_PATH}}, help_text=_( - 'Arguments to pass to the drivers.' - ) -) setting_auto_process = namespace.add_setting( global_name='FILE_METADATA_AUTO_PROCESS', default=True, help_text=_( @@ -21,3 +15,13 @@ setting_auto_process = namespace.add_setting( 'automatically by default.' ) ) +setting_drivers_arguments = namespace.add_setting( + default=''' + {{ + exif_driver: {{exiftool_path: {}}}, + + }} + '''.replace('\n', '').format(DEFAULT_EXIF_PATH), help_text=_( + 'Arguments to pass to the drivers.' + ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS', quoted=True +) diff --git a/mayan/apps/file_metadata/tests/literals.py b/mayan/apps/file_metadata/tests/literals.py index 295cb5a4b0..3b9e6e9a33 100644 --- a/mayan/apps/file_metadata/tests/literals.py +++ b/mayan/apps/file_metadata/tests/literals.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals TEST_DRIVER_INTERNAL_NAME = 'exiftool' TEST_FILE_METADATA_KEY = 'FileType' TEST_FILE_METADATA_VALUE = 'PNG' -TEST_FILE_METADATA_INDEX_NODE_TEMPLATE = "{{{{ document.get_file_metadata('{}.{}')}}}}".format( +TEST_FILE_METADATA_INDEX_NODE_TEMPLATE = "{{{{ document.file_metadata_value_of.{}_{} }}}}".format( TEST_DRIVER_INTERNAL_NAME, TEST_FILE_METADATA_KEY ) TEST_PDF_FILE_METADATA_DOTTED_NAME = 'exiftool.Producer' diff --git a/mayan/apps/file_metadata/tests/test_indexing.py b/mayan/apps/file_metadata/tests/test_indexing.py index be0a0b474a..441a4e39ca 100644 --- a/mayan/apps/file_metadata/tests/test_indexing.py +++ b/mayan/apps/file_metadata/tests/test_indexing.py @@ -16,17 +16,18 @@ class IndexingTestCase(DocumentTestMixin, BaseTestCase): def test_indexing(self): index = Index.objects.create(label=TEST_INDEX_LABEL) - index.document_types.add(self.document_type) + index.document_types.add(self.test_document_type) root = index.template_root index.node_templates.create( parent=root, expression=TEST_FILE_METADATA_INDEX_NODE_TEMPLATE, link_documents=True ) - self.document = self.upload_document() - self.document.submit_for_file_metadata_processing() + self.upload_document() + self.test_document.submit_for_file_metadata_processing() + index.rebuild() self.assertTrue( - self.document in IndexInstanceNode.objects.get( + self.test_document in IndexInstanceNode.objects.get( value=TEST_FILE_METADATA_VALUE ).documents.all() ) diff --git a/mayan/apps/file_metadata/tests/test_views.py b/mayan/apps/file_metadata/tests/test_views.py index d2dbe87add..81376db3ee 100644 --- a/mayan/apps/file_metadata/tests/test_views.py +++ b/mayan/apps/file_metadata/tests/test_views.py @@ -14,6 +14,10 @@ from .literals import TEST_FILE_METADATA_KEY @override_settings(FILE_METADATA_AUTO_PROCESS=True) class FileMetadataViewsTestCase(GenericDocumentViewTestCase): + def setUp(self): + super(FileMetadataViewsTestCase, self).setUp() + self.test_driver = self.document.latest_version.file_metadata_drivers.first() + def _request_document_version_driver_list_view(self): return self.get( viewname='file_metadata:document_driver_list', @@ -28,6 +32,7 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): self.grant_access( permission=permission_file_metadata_view, obj=self.document ) + response = self._request_document_version_driver_list_view() self.assertContains( response=response, text=self.document.label, status_code=200 @@ -36,7 +41,9 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): def _request_document_version_file_metadata_list_view(self): return self.get( viewname='file_metadata:document_version_driver_file_metadata_list', - kwargs={'document_version_driver_id': self.document.latest_version.file_metadata_drivers.first().pk} + kwargs={ + 'document_version_driver_id': self.test_driver.pk + } ) def test_document_version_file_metadata_list_view_no_permission(self): @@ -49,6 +56,7 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): self.grant_access( obj=self.document, permission=permission_file_metadata_view ) + response = self._request_document_version_file_metadata_list_view() self.assertContains( response=response, text=TEST_FILE_METADATA_KEY, status_code=200 @@ -62,8 +70,10 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): def test_document_submit_view_no_permission(self): self.document.latest_version.file_metadata_drivers.all().delete() + response = self._request_document_submit_view() - self.assertEqual(response.status_code, 404) + self.assertEqual(response.status_code, 302) + self.assertEqual( self.document.latest_version.file_metadata_drivers.count(), 0 ) @@ -73,8 +83,10 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): self.grant_access( permission=permission_file_metadata_submit, obj=self.document ) + response = self._request_document_submit_view() self.assertEqual(response.status_code, 302) + self.assertEqual( self.document.latest_version.file_metadata_drivers.count(), 1 ) @@ -89,8 +101,10 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): def test_multiple_document_submit_view_no_permission(self): self.document.latest_version.file_metadata_drivers.all().delete() + response = self._request_multiple_document_submit_view() - self.assertEqual(response.status_code, 404) + self.assertEqual(response.status_code, 302) + self.assertEqual( self.document.latest_version.file_metadata_drivers.count(), 0 ) @@ -100,18 +114,16 @@ class FileMetadataViewsTestCase(GenericDocumentViewTestCase): self.grant_access( permission=permission_file_metadata_submit, obj=self.document ) + response = self._request_multiple_document_submit_view() self.assertEqual(response.status_code, 302) + self.assertEqual( self.document.latest_version.file_metadata_drivers.count(), 1 ) class DocumentTypeViewsTestCase(GenericDocumentViewTestCase): - def setUp(self): - super(DocumentTypeViewsTestCase, self).setUp() - self.login_user() - def _request_document_type_settings_view(self): return self.get( viewname='file_metadata:document_type_settings', @@ -124,11 +136,11 @@ class DocumentTypeViewsTestCase(GenericDocumentViewTestCase): def test_document_type_settings_view_with_access(self): self.grant_access( - permission=permission_document_type_file_metadata_setup, - obj=self.document_type + obj=self.document_type, + permission=permission_document_type_file_metadata_setup ) - response = self._request_document_type_settings_view() + response = self._request_document_type_settings_view() self.assertEqual(response.status_code, 200) def _request_document_type_submit_view(self): @@ -141,16 +153,19 @@ class DocumentTypeViewsTestCase(GenericDocumentViewTestCase): def test_document_type_submit_view_no_permission(self): response = self._request_document_type_submit_view() self.assertEqual(response.status_code, 200) + self.assertEqual( self.document.latest_version.file_metadata_drivers.count(), 0 ) def test_document_type_submit_view_with_access(self): self.grant_access( - obj=self.document_type, permission=permission_file_metadata_submit, + obj=self.document_type, permission=permission_file_metadata_submit ) + response = self._request_document_type_submit_view() self.assertEqual(response.status_code, 302) + self.assertEqual( self.document.latest_version.file_metadata_drivers.count(), 1 ) diff --git a/mayan/apps/file_metadata/views.py b/mayan/apps/file_metadata/views.py index 7c3a6fbf10..8456917d88 100644 --- a/mayan/apps/file_metadata/views.py +++ b/mayan/apps/file_metadata/views.py @@ -143,12 +143,12 @@ class DocumentTypeSubmitView(FormView): count += 1 messages.success( - self.request, _( + message=_( '%(count)d documents added to the file metadata processing ' 'queue.' ) % { 'count': count, - } + }, request=self.request ) - return HttpResponseRedirect(self.get_success_url()) + return HttpResponseRedirect(redirect_to=self.get_success_url())