diff --git a/HISTORY.rst b/HISTORY.rst index 41a65db9e4..e9dc7a1ca2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,8 @@ 3.2 (2019-04-XX) ================ * Split sources models into separate modules +* Add support for subfolder scanning to watchfolders. Closes + GitLab issue #498 and #563. 3.1.10 (2019-04-04) =================== diff --git a/docs/releases/3.2.rst b/docs/releases/3.2.rst new file mode 100644 index 0000000000..ef298bf127 --- /dev/null +++ b/docs/releases/3.2.rst @@ -0,0 +1,87 @@ +Version 3.2 +=========== + +Released: XX XX, 2019 + + +Changes +------- + + +Other changes +^^^^^^^^^^^^^ + +* Split source models into different modules. + +Removals +-------- + +* None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + +Type in the console:: + + $ pip install mayan-edms==3.2 + +the requirements will also be updated automatically. + + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +^^^^^^^^^^^^ + +Perform these steps after updating the code from either step above. + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py collectstatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +* None + + +Bugs fixed or issues closed +--------------------------- + +* :gitlab-issue:`498` Can't scan subdirectories +* :gitlab-issue:`563` Recursive Watch Folder + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index 53fa556139..634f088cf2 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -137,6 +137,6 @@ class WatchFolderSetupForm(forms.ModelForm): class Meta: fields = ( 'label', 'enabled', 'interval', 'document_type', 'uncompress', - 'folder_path' + 'folder_path', 'include_subdirectories' ) model = WatchFolderSource diff --git a/mayan/apps/sources/migrations/0020_auto_20181128_0752.py b/mayan/apps/sources/migrations/0020_auto_20181128_0752.py new file mode 100644 index 0000000000..49ecd7bbbf --- /dev/null +++ b/mayan/apps/sources/migrations/0020_auto_20181128_0752.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2018-11-28 07:52 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0019_auto_20180803_0440'), + ] + + operations = [ + migrations.AddField( + model_name='watchfoldersource', + name='include_subdirectories', + field=models.BooleanField(default=False, help_text='If checked, not only will the folder path be scanned for files but also its subdirectories.', verbose_name='Include subdirectories?'), + preserve_default=False, + ), + migrations.AlterField( + model_name='watchfoldersource', + name='folder_path', + field=models.CharField(help_text='Server side filesystem path to scan for files.', max_length=255, verbose_name='Folder path'), + ), + ] diff --git a/mayan/apps/sources/models/watch_folder_sources.py b/mayan/apps/sources/models/watch_folder_sources.py index 2a45ddf530..07d0b8e5ac 100644 --- a/mayan/apps/sources/models/watch_folder_sources.py +++ b/mayan/apps/sources/models/watch_folder_sources.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals import logging -import os -from django.core.files import File +from pathlib2 import Path + from django.db import models -from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ from ..literals import SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICE_Y @@ -30,8 +29,15 @@ class WatchFolderSource(IntervalBaseModel): source_type = SOURCE_CHOICE_WATCH folder_path = models.CharField( - help_text=_('Server side filesystem path.'), max_length=255, - verbose_name=_('Folder path') + help_text=_('Server side filesystem path to scan for files.'), + max_length=255, verbose_name=_('Folder path') + ) + include_subdirectories = models.BooleanField( + help_text=_( + 'If checked, not only will the folder path be scanned for files ' + 'but also its subdirectories.' + ), + verbose_name=_('Include subdirectories?') ) objects = models.Manager() @@ -41,15 +47,19 @@ class WatchFolderSource(IntervalBaseModel): verbose_name_plural = _('Watch folders') def check_source(self): - # Force self.folder_path to unicode to avoid os.listdir returning - # str for non-latin filenames, gh-issue #163 - for file_name in os.listdir(force_text(self.folder_path)): - full_path = os.path.join(self.folder_path, file_name) - if os.path.isfile(full_path): - with File(file=open(full_path, mode='rb')) as file_object: + path = Path(self.folder_path) + + if self.include_subdirectories: + iterator = path.rglob('*') + else: + iterator = path.glob('*') + + for entry in iterator: + if entry.is_file() or entry.is_symlink(): + with entry.open(mode='rb') as file_object: self.handle_upload( file_object=file_object, expand=(self.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), - label=file_name + label=entry.name ) - os.unlink(full_path) + entry.unlink() diff --git a/mayan/apps/sources/tests/literals.py b/mayan/apps/sources/tests/literals.py index 64be46beb9..5ab77120db 100644 --- a/mayan/apps/sources/tests/literals.py +++ b/mayan/apps/sources/tests/literals.py @@ -137,3 +137,4 @@ Content-MD5: 1B2M2Y8AsgTpgAmY7PhCfg== TEST_SOURCE_LABEL = 'test source' TEST_SOURCE_UNCOMPRESS_N = 'n' TEST_STAGING_PREVIEW_WIDTH = 640 +TEST_WATCHFOLDER_SUBFOLDER = 'test_subfolder' diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 8f8c52b6bd..05a6c4c0e8 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -3,29 +3,31 @@ from __future__ import unicode_literals import shutil import mock +from pathlib2 import Path from django.test import override_settings +from django.utils.encoding import force_text from common.utils import mkdtemp from common.tests import BaseTestCase from documents.models import Document, DocumentType from documents.tests import ( - TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL, + DocumentTestMixin, TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL, TEST_NON_ASCII_DOCUMENT_FILENAME, TEST_NON_ASCII_DOCUMENT_PATH, TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH ) from metadata.models import MetadataType from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y -from ..models import ( - EmailBaseModel, POP3Email, WatchFolderSource, WebFormSource -) +from ..models import POP3Email, WatchFolderSource, WebFormSource +from ..models.email_sources import EmailBaseModel from .literals import ( TEST_EMAIL_ATTACHMENT_AND_INLINE, TEST_EMAIL_BASE64_FILENAME, TEST_EMAIL_BASE64_FILENAME_FROM, TEST_EMAIL_BASE64_FILENAME_SUBJECT, TEST_EMAIL_INLINE_IMAGE, TEST_EMAIL_NO_CONTENT_TYPE, - TEST_EMAIL_NO_CONTENT_TYPE_STRING, TEST_EMAIL_ZERO_LENGTH_ATTACHMENT + TEST_EMAIL_NO_CONTENT_TYPE_STRING, TEST_EMAIL_ZERO_LENGTH_ATTACHMENT, + TEST_WATCHFOLDER_SUBFOLDER ) @@ -244,35 +246,47 @@ class POP3SourceTestCase(BaseTestCase): @override_settings(OCR_AUTO_OCR=False) -class UploadDocumentTestCase(BaseTestCase): - """ - Test creating documents - """ - def setUp(self): - super(UploadDocumentTestCase, self).setUp() - self.document_type = DocumentType.objects.create( - label=TEST_DOCUMENT_TYPE_LABEL - ) +class WatchFolderTestCase(DocumentTestMixin, BaseTestCase): + auto_upload_document = False - def tearDown(self): - self.document_type.delete() - super(UploadDocumentTestCase, self).tearDown() - - def test_issue_gh_163(self): - """ - Non-ASCII chars in document name failing in upload via watch folder - gh-issue #163 https://github.com/mayan-edms/mayan-edms/issues/163 - """ - - temporary_directory = mkdtemp() - shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory) - - watch_folder = WatchFolderSource.objects.create( - document_type=self.document_type, folder_path=temporary_directory, + def _create_watchfolder(self): + return WatchFolderSource.objects.create( + document_type=self.document_type, + folder_path=self.temporary_directory, + include_subdirectories=False, uncompress=SOURCE_UNCOMPRESS_CHOICE_Y ) - watch_folder.check_source() + def setUp(self): + super(WatchFolderTestCase, self).setUp() + self.temporary_directory = mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temporary_directory) + super(WatchFolderTestCase, self).tearDown() + + def test_subfolder_support_disabled(self): + watch_folder = self._create_watchfolder() + + test_path = Path(self.temporary_directory) + test_subfolder = test_path.joinpath(TEST_WATCHFOLDER_SUBFOLDER) + test_subfolder.mkdir() + + shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, force_text(test_subfolder)) + watch_folder.check_source() + self.assertEqual(Document.objects.count(), 0) + + def test_subfolder_support_enabled(self): + watch_folder = self._create_watchfolder() + watch_folder.include_subdirectories = True + watch_folder.save() + + test_path = Path(self.temporary_directory) + test_subfolder = test_path.joinpath(TEST_WATCHFOLDER_SUBFOLDER) + test_subfolder.mkdir() + + shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, force_text(test_subfolder)) + watch_folder.check_source() self.assertEqual(Document.objects.count(), 1) document = Document.objects.first() @@ -285,16 +299,18 @@ class UploadDocumentTestCase(BaseTestCase): self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) self.assertEqual(document.page_count, 1) - # Test Non-ASCII named documents inside Non-ASCII named compressed file - - shutil.copy( - TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH, temporary_directory - ) + def test_issue_gh_163(self): + """ + Non-ASCII chars in document name failing in upload via watch folder + gh-issue #163 https://github.com/mayan-edms/mayan-edms/issues/163 + """ + watch_folder = self._create_watchfolder() + shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, self.temporary_directory) watch_folder.check_source() - document = Document.objects.all()[1] + self.assertEqual(Document.objects.count(), 1) - self.assertEqual(Document.objects.count(), 2) + document = Document.objects.first() self.assertEqual(document.exists(), True) self.assertEqual(document.size, 17436) @@ -304,4 +320,24 @@ class UploadDocumentTestCase(BaseTestCase): self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) self.assertEqual(document.page_count, 1) - shutil.rmtree(temporary_directory) + def test_issue_gh_163_expanded(self): + """ + Test Non-ASCII named documents inside Non-ASCII named compressed file + """ + watch_folder = self._create_watchfolder() + + shutil.copy( + TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH, self.temporary_directory + ) + watch_folder.check_source() + self.assertEqual(Document.objects.count(), 1) + + document = Document.objects.first() + + self.assertEqual(document.exists(), True) + self.assertEqual(document.size, 17436) + + self.assertEqual(document.file_mimetype, 'image/png') + self.assertEqual(document.file_mime_encoding, 'binary') + self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) + self.assertEqual(document.page_count, 1)