Document zip file encoding issues

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

HISTORY.rst

@@ -1,4 +1,4 @@
-3.2.8 (2019-10-01)
+3.2.8 (2019-XX-XX)
 ==================
 - Fix error when accessing some API entry points without
   being authenticated.

docker/rootfs/version

@@ -1 +1 @@
-3.2.8
+3.2.7

docs/chapters/development.rst

@@ -533,7 +533,7 @@ Release using GitLab CI
    ::
 
        git checkout releases/all
-       git merge <corresponding branch>
+       git merge versions/next
 
 #. Push code to trigger builds:
    ::

docs/releases/3.2.8.rst

@@ -1,21 +1,12 @@
 Version 3.2.8
 =============
 
-Released: October 1, 2019
+Released: XX, 2019
 
 
 Changes
 -------
 
-
-API
-^^^
-
-Fix an error when accessing some API entry points without
-being authenticated. Accessing API endpoints without being authenticated
-will now always return empty results.
-
-
 Cabinets
 ^^^^^^^^
 
@@ -23,43 +14,26 @@ Tweaked the jstree component's appearance to cope with long labels.
 Added a scrollbar, reduced the font size, switched to a sans serif font,
 and reduced padding. Thanks for forum user @briboe for the report.
 
-Workflow actions to add and remove documents from cabinets was added.
 
+Other changes
+^^^^^^^^^^^^^
 
-Dependencies
+- Fix error when accessing some API entry points without
-^^^^^^^^^^^^
+  being authenticated.
-
+- Add cabinet add and remove workflow actions.
-The Django version used was updated to version 1.11.24. The jQuery version
+- Update Django to version 1.11.24.
-used was updated to version 3.4.1. Both as fully backwards compatible with
+- Update jQuery to version 3.4.1
-their previous versions.
+- Add support for deleting the OCR content of a document
-
+  or selection of documents.
-
+- Add OCR content deleted event.
-OCR
+- Add missing recursive option to Docker entrypoint
-^^^
+  chown. GitLab issue #668. Thanks to John Wice (@brilthor)
-
-Support was added to delete the content of document's OCR or parsed content.
-Events for both situations was added allowing content deletion to be used
-as workflow transition triggers.
-
-
-Docker
-^^^^^^
-
-A missing recursive option was added to the Docker entrypoint
-command "chown" to change the ownership of files when specifying a custom
-UID or GID. Closes GitLab issue #668. Thanks to John Wice (@brilthor)
   for the report.
-
+- Add support for deleting the parsed content of a document
-Two fonts were added to the Docker image to support rendering Chinese office
+  of selection of documents.
-documents. Closes GitLab issue #666. Thanks to javawcy (@javawcy) and forum
+- Add parsed content deleted event.
-user @leoliu for the report and help closing this issue.
+- Allow scaling of UI on mobile devices.
-
+- Add Chinese fonts to the Docker image
-
-Usability
-^^^^^^^^^
-
-Descriptions for screenreaders was added via image alt tag. The user interface
-will also now allow scaling.
 
 
 Removals

mayan/__init__.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '3.2.8'
+__version__ = '3.2.7'
-__build__ = 0x030208
+__build__ = 0x030207
-__build_string__ = 'v3.2.8_Tue Oct 1 13:31:40 2019 -0400'
+__build_string__ = 'v3.2.7_Wed Aug 28 17:31:08 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'

mayan/apps/common/compressed_files.py

@@ -11,6 +11,7 @@ except ImportError:
     COMPRESSION = zipfile.ZIP_STORED
 
 from django.core.files.uploadedfile import SimpleUploadedFile
+from django.utils.encoding import force_text
 
 from mayan.apps.mimetype.api import get_mimetype
 
@@ -136,9 +137,32 @@ class ZipArchive(Archive):
         return self._archive.read(filename)
 
     def members(self):
-        return [
+        results = []
-            filename for filename in self._archive.namelist() if not filename.endswith('/')
+        from django.utils.encoding import force_str
-        ]
+
+        for filename in self._archive.namelist():
+            # Zip files only support UTF-8 and CP437 encodings.
+            # Attempt to decode CP437 to be able to check if it ends
+            # with a slash.
+            # Future improvement that violates the Zip format:
+            # Add chardet.detect to detect the most likely encoding
+            # if other than CP437.
+            try:
+                filename = filename.decode('CP437')
+                is_unicode = False
+            except UnicodeEncodeError:
+                is_unicode = True
+
+            if not filename.endswith('/'):
+                # Re encode in the original encoding
+                if not is_unicode:
+                    filename = filename.encode(
+                        encoding='CP437', errors='strict'
+                    )
+
+                results.append(filename)
+
+        return results
 
     def open_member(self, filename):
         return self._archive.open(filename)

mayan/apps/common/tests/literals.py

@@ -12,6 +12,7 @@ TEST_VIEW_URL = 'test-view-url'
 
 # Filenames
 TEST_ARCHIVE_ZIP_SPECIAL_CHARACTERS_FILENAME_MEMBER = 'test_archvive_with_special_characters_filename_member.zip'
+TEST_ARCHIVE_ZIP_CP437_MEMBER = 'test_archvive_with_cp437_member.zip'
 TEST_FILENAME1 = 'test_file1.txt'
 TEST_FILENAME2 = 'test_file2.txt'
 TEST_FILENAME3 = 'test_file3.txt'
@@ -28,6 +29,10 @@ TEST_ARCHIVE_ZIP_SPECIAL_CHARACTERS_FILENAME_MEMBER_PATH = os.path.join(
     settings.BASE_DIR, 'apps', 'common', 'tests', 'contrib',
     TEST_ARCHIVE_ZIP_SPECIAL_CHARACTERS_FILENAME_MEMBER
 )
+TEST_ARCHIVE_ZIP_CP437_MEMBER_PATH = os.path.join(
+    settings.BASE_DIR, 'apps', 'common', 'tests', 'contrib',
+    TEST_ARCHIVE_ZIP_CP437_MEMBER
+)
 TEST_FILE3_PATH = os.path.join(
     settings.BASE_DIR, 'apps', 'common', 'tests', 'contrib', TEST_FILENAME3
 )

mayan/apps/common/tests/test_compressed_files.py

@@ -5,6 +5,7 @@ from mayan.apps.common.tests import BaseTestCase
 from ..compressed_files import Archive, TarArchive, ZipArchive
 
 from .literals import (
+    TEST_ARCHIVE_ZIP_CP437_MEMBER_PATH,
     TEST_ARCHIVE_ZIP_SPECIAL_CHARACTERS_FILENAME_MEMBER_PATH,
     TEST_COMPRESSED_FILE_CONTENTS, TEST_FILE_CONTENTS_1, TEST_FILE3_PATH,
     TEST_FILENAME1, TEST_FILENAME3, TEST_TAR_BZ2_FILE_PATH,
@@ -64,6 +65,11 @@ class ZipArchiveClassTestCase(TarArchiveClassTestCase):
             archive = Archive.open(file_object=file_object)
             list(archive.get_members())
 
+    def test_open_cp437_member(self):
+        with open(TEST_ARCHIVE_ZIP_CP437_MEMBER_PATH, mode='rb') as file_object:
+            archive = Archive.open(file_object=file_object)
+            list(archive.get_members())
+
 
 class TarGzArchiveClassTestCase(TarArchiveClassTestCase):
     archive_path = TEST_TAR_GZ_FILE_PATH
@@ -73,3 +79,5 @@ class TarGzArchiveClassTestCase(TarArchiveClassTestCase):
 class TarBz2ArchiveClassTestCase(TarArchiveClassTestCase):
     archive_path = TEST_TAR_BZ2_FILE_PATH
     cls = TarArchive
+
+

mayan/apps/document_states/widgets.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 from django import forms
 from django.urls import reverse
 from django.utils.html import format_html_join, mark_safe
-from django.utils.translation import ugettext_lazy as _
 
 
 def widget_transition_events(transition):

mayan/apps/linking/tests/mixins.py

@@ -10,14 +10,6 @@ from .literals import (
 )
 
 
-class SmartLinkDocumentViewTestMixin(object):
-    def _request_test_smart_link_document_instances_view(self):
-        return self.get(
-            viewname='linking:smart_link_instances_for_document',
-            kwargs={'pk': self.test_document.pk}
-        )
-
-
 class SmartLinkTestMixin(object):
     def _create_test_smart_link(self, add_test_document_type=False):
         self.test_smart_link = SmartLink.objects.create(

mayan/apps/linking/tests/test_views.py

@@ -14,15 +14,10 @@ from .literals import (
     TEST_SMART_LINK_DYNAMIC_LABEL, TEST_SMART_LINK_LABEL_EDITED,
     TEST_SMART_LINK_LABEL
 )
-from .mixins import (
+from .mixins import SmartLinkTestMixin, SmartLinkViewTestMixin
-    SmartLinkDocumentViewTestMixin, SmartLinkTestMixin,
-    SmartLinkViewTestMixin
-)
 
 
-class SmartLinkViewTestCase(
+class SmartLinkViewTestCase(SmartLinkTestMixin, SmartLinkViewTestMixin, GenericViewTestCase):
-    SmartLinkTestMixin, SmartLinkViewTestMixin, GenericViewTestCase
-):
     def test_smart_link_create_view_no_permission(self):
         response = self._request_test_smart_link_create_view()
         self.assertEqual(response.status_code, 403)
@@ -79,15 +74,10 @@ class SmartLinkViewTestCase(
         self.assertEqual(response.status_code, 302)
 
         self.test_smart_link.refresh_from_db()
-        self.assertEqual(
+        self.assertEqual(self.test_smart_link.label, TEST_SMART_LINK_LABEL_EDITED)
-            self.test_smart_link.label, TEST_SMART_LINK_LABEL_EDITED
-        )
 
 
-class SmartLinkDocumentViewTestCase(
+class SmartLinkDocumentViewTestCase(SmartLinkTestMixin, GenericDocumentViewTestCase):
-    SmartLinkTestMixin, SmartLinkDocumentViewTestMixin,
-    GenericDocumentViewTestCase
-):
     def setUp(self):
         super(SmartLinkDocumentViewTestCase, self).setUp()
         self._create_test_smart_link()
@@ -99,6 +89,12 @@ class SmartLinkDocumentViewTestCase(
         )
         self.test_smart_link_2.document_types.add(self.test_document_type)
 
+    def _request_test_smart_link_document_instances_view(self):
+        return self.get(
+            viewname='linking:smart_link_instances_for_document',
+            kwargs={'pk': self.test_document.pk}
+        )
+
     def test_document_smart_link_list_view_no_permission(self):
         self.grant_access(
             obj=self.test_document, permission=permission_document_view

setup.py

@@ -56,7 +56,7 @@ def find_packages(directory):
     return packages
 
 install_requires = """
-django==1.11.24
+django==1.11.22
 Pillow==6.0.0
 PyPDF2==1.26.0
 PyYAML==5.1.1