Merge branch 'master' into feature/tornado

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -1,3 +1,11 @@
+2.7.3 (2017-09-11)
+==================
+- Fix task manager queue list view. Thanks to LeVon Smoker for
+  the report.
+- Fix resolved link class URL mangling when the keep_query argument is
+  used. Thanks to Nick Douma (LordGaav) for the report and diagnostic
+  information. Fixes source navigation on the document upload wizard.
+
 2.7.2 (2017-09-06)
 ==================
 - Fix new mailer creation view. GitLab issue #431.
@@ -352,7 +360,6 @@
 - Add roadmap documentation chapter.
 - API updates.
 
-
 2.0.2 (2016-02-09)
 ==================
 - Install testing dependencies when installing development dependencies.

docs/releases/2.1.rst

@@ -108,6 +108,15 @@ screen. These messages can have an activation and an expiration date and
 time. These messages are useful for display company access policies,
 maintenance announcement, etc.
 
+New server app using tornado
+----------------------------
+This release includes a simple app that can serve Mayan EDMS using the tornado
+server. Using this app users can start using Mayan EDMS with minimal setup
+(just install Redis). By default the server will run on port 52723, but users
+can change the port with the --port option. For privileged port (ports
+below 1024) the command must be run as superadmin. This is an experimental
+feature, feedback and patches are appreciated.
+
 Document signing
 ----------------
 The biggest change for this release if the addition of document signing from

docs/releases/2.7.3.rst

@@ -0,0 +1,74 @@
+===============================
+Mayan EDMS v2.7.3 release notes
+===============================
+
+Released: September 11, 2017
+
+What's new
+==========
+
+- Fix task manager queue list view. Thanks to LeVon Smoker for
+  the report.
+- Fix resolved link class URL mangling when the keep_query argument is
+  used. Fixes source navigation on the document upload wizard. Thanks to
+  Nick Douma (LordGaav) for the report and diagnostic information. GitLab
+  issue #436.
+
+Removals
+--------
+* None
+
+Upgrading from a previous version
+---------------------------------
+
+Using PIP
+~~~~~~~~~
+
+Type in the console::
+
+    $ pip install -U mayan-edms
+
+the requirements will also be updated automatically.
+
+Using Git
+~~~~~~~~~
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+Common steps
+~~~~~~~~~~~~
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+=============================
+
+* None
+
+Bugs fixed or issues closed
+===========================
+
+* `GitLab issue #431 <https://gitlab.com/mayan-edms/mayan-edms/issues/431>`_ can't create new mailer
+* `GitLab issue #436 <https://gitlab.com/mayan-edms/mayan-edms/issues/436>`_ New document source menu does not contain source_ids
+
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -22,6 +22,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   2.7.3
    2.7.2
    2.7.1
    2.7

mayan/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '2.7.2'
+__version__ = '2.7.3'
-__build__ = 0x020702
+__build__ = 0x020703
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'

mayan/apps/document_signatures/apps.py

@@ -135,7 +135,7 @@ class DocumentSignaturesApp(MayanAppConfig):
         menu_facet.bind_links(
             links=(
                 link_document_version_signature_list,
-            ), sources=(DocumentVersion,)
+            ), position=9, sources=(DocumentVersion,)
         )
 
         menu_object.bind_links(

mayan/apps/documents/apps.py

@@ -437,9 +437,6 @@ class DocumentsApp(MayanAppConfig):
             sources=(Document,), position=2
         )
         menu_facet.bind_links(links=(link_document_pages,), sources=(Document,))
-        menu_facet.bind_links(
-            links=(link_document_version_view,), sources=(DocumentVersion,)
-        )
 
         # Document actions
         menu_object.bind_links(
@@ -493,6 +490,9 @@ class DocumentsApp(MayanAppConfig):
                 link_document_version_return_list
             ), sources=(DocumentVersion,)
         )
+        menu_facet.bind_links(
+            links=(link_document_version_view,), sources=(DocumentVersion,)
+        )
 
         namespace = StatisticNamespace(slug='documents', label=_('Documents'))
         namespace.add_statistic(

mayan/apps/kaze/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'kaze.apps.KazeApp'

mayan/apps/kaze/apps.py

@@ -0,0 +1,13 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from common import MayanAppConfig
+
+
+class KazeApp(MayanAppConfig):
+    name = 'kaze'
+    verbose_name = _('Kaze')
+
+    def ready(self):
+        super(KazeApp, self).ready()

mayan/apps/kaze/management/commands/serve.py

@@ -0,0 +1,69 @@
+from __future__ import unicode_literals
+
+import os
+
+from django.core import management
+from django.core.wsgi import get_wsgi_application
+
+import tornado.httpserver
+import tornado.ioloop
+from tornado.process import Subprocess
+import tornado.web
+import tornado.wsgi
+
+DEFAULT_PORT = 8080
+
+
+class Command(management.BaseCommand):
+    help = 'Launches a local Tornado server.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--single-process',
+            action='store_true',
+            dest='single-process',
+            default=False,
+            help='Forces only one server process.'
+        )
+
+        parser.add_argument(
+            '--port',
+            action='store',
+            dest='port',
+            default=DEFAULT_PORT,
+            help='Port on which to bind the server.'
+        )
+
+    def handle(self, *args, **options):
+        wsgi_application = get_wsgi_application()
+        wsgi_container = tornado.wsgi.WSGIContainer(wsgi_application)
+
+        tornado_application = tornado.web.Application(
+            handlers=(
+                (
+                    r'/static/(.*)', tornado.web.StaticFileHandler,
+                    {'path': 'mayan/media/static'},
+                ),
+                (
+                    '.*', tornado.web.FallbackHandler,
+                    dict(fallback=wsgi_container)
+                ),
+            )
+        )
+
+        http_server = tornado.httpserver.HTTPServer(tornado_application)
+
+        try:
+            if options['single-process']:
+                http_server.listen(options['port'])
+                ioloop = tornado.ioloop.IOLoop.instance()
+                Subprocess(['./manage.py', 'celery', 'worker', '-O', 'fair'])
+                ioloop.start()
+            else:
+                http_server.bind(options['port'])
+                http_server.start(0)  # forks one process per cpu
+                ioloop = tornado.ioloop.IOLoop.current()
+                Subprocess(['./manage.py', 'celery', 'worker', '-O', 'fair'])
+                ioloop.start()
+        except KeyboardInterrupt:
+            tornado.ioloop.IOLoop.instance().stop()

mayan/apps/navigation/classes.py

@@ -363,7 +363,10 @@ class Link(object):
                 except KeyError:
                     pass
 
-            resolved_link.url = parsed_url.url
+            # Use the link's URL but with the previous URL querystring
+            new_url = furl(resolved_link.url)
+            new_url.args = parsed_url.querystr
+            resolved_link.url = new_url.url
 
         resolved_link.context = context
         return resolved_link

mayan/apps/navigation/tests/literals.py

@@ -7,5 +7,8 @@ TEST_PERMISSION_NAME = 'test permission name'
 TEST_PERMISSION_LABEL = 'test permission label'
 TEST_LINK_TEXT = 'test link text'
 TEST_MENU_NAME = 'menu test'
+TEST_QUERYSTRING_ONE_KEY = 'key1=value1'
+TEST_QUERYSTRING_TWO_KEYS = 'key1=value1&key2=value2'
 TEST_SUBMENU_NAME = 'submenu test'
 TEST_UNICODE_STRING = 'úñí©óðé'
+TEST_URL = 'test-URL'

mayan/apps/navigation/tests/test_classes.py

@@ -15,7 +15,8 @@ from ..classes import Link, Menu
 from .literals import (
     TEST_PERMISSION_NAMESPACE_NAME, TEST_PERMISSION_NAMESPACE_TEXT,
     TEST_PERMISSION_NAME, TEST_PERMISSION_LABEL, TEST_LINK_TEXT,
-    TEST_MENU_NAME, TEST_SUBMENU_NAME, TEST_UNICODE_STRING
+    TEST_MENU_NAME, TEST_QUERYSTRING_ONE_KEY, TEST_QUERYSTRING_TWO_KEYS,
+    TEST_SUBMENU_NAME, TEST_UNICODE_STRING, TEST_URL
 )
 
 
@@ -115,6 +116,43 @@ class LinkClassTestCase(GenericViewTestCase):
         self.assertEqual(resolved_link.url, url.url)
 
 
+    def test_link_with_querystring_preservation(self):
+        previous_url = '{}?{}'.format(
+            reverse(TEST_VIEW_NAME), TEST_QUERYSTRING_TWO_KEYS
+        )
+        self.link.keep_query = True
+        self.link.url = TEST_URL
+        self.link.view = None
+        response = self.get(path=previous_url)
+
+        context = Context({'request': response.wsgi_request})
+
+        resolved_link = self.link.resolve(context=context)
+
+        self.assertEqual(
+            resolved_link.url,
+            '{}?{}'.format(TEST_URL, TEST_QUERYSTRING_TWO_KEYS)
+        )
+
+    def test_link_with_querystring_preservation_with_key_removal(self):
+        previous_url = '{}?{}'.format(
+            reverse(TEST_VIEW_NAME), TEST_QUERYSTRING_TWO_KEYS
+        )
+        self.link.keep_query = True
+        self.link.url = TEST_URL
+        self.link.view = None
+        self.link.remove_from_query = ['key2']
+        response = self.get(path=previous_url)
+
+        context = Context({'request': response.wsgi_request})
+
+        resolved_link = self.link.resolve(context=context)
+        self.assertEqual(
+            resolved_link.url,
+            '{}?{}'.format(TEST_URL, TEST_QUERYSTRING_ONE_KEY)
+        )
+
+
 class MenuClassTestCase(GenericViewTestCase):
     def setUp(self):
         super(MenuClassTestCase, self).setUp()

mayan/apps/sources/tests/literals.py

@@ -0,0 +1,5 @@
+from __future__ import unicode_literals
+
+TEST_SOURCE_LABEL = 'test source'
+TEST_SOURCE_UNCOMPRESS_N = 'n'
+TEST_STAGING_PREVIEW_WIDTH = 640

mayan/apps/sources/tests/test_views.py

@@ -29,9 +29,9 @@ from ..permissions import (
     permission_sources_setup_view, permission_staging_file_delete
 )
 
-TEST_SOURCE_LABEL = 'test source'
+from .literals import (
-TEST_SOURCE_UNCOMPRESS_N = 'n'
+    TEST_SOURCE_LABEL, TEST_SOURCE_UNCOMPRESS_N, TEST_STAGING_PREVIEW_WIDTH
-TEST_STAGING_PREVIEW_WIDTH = 640
+)
 
 
 class DocumentUploadTestCase(GenericDocumentViewTestCase):
@@ -253,9 +253,9 @@ class NewDocumentVersionViewTestCase(GenericDocumentViewTestCase):
         self.assertEqual(resolved_link, None)
 
 
-class StagingFolderTestCase(GenericViewTestCase):
+class StagingFolderViewTestCase(GenericViewTestCase):
     def setUp(self):
-        super(StagingFolderTestCase, self).setUp()
+        super(StagingFolderViewTestCase, self).setUp()
         self.temporary_directory = mkdtemp()
         shutil.copy(TEST_SMALL_DOCUMENT_PATH, self.temporary_directory)
 
@@ -263,7 +263,7 @@ class StagingFolderTestCase(GenericViewTestCase):
 
     def tearDown(self):
         fs_cleanup(self.temporary_directory)
-        super(StagingFolderTestCase, self).tearDown()
+        super(StagingFolderViewTestCase, self).tearDown()
 
     def test_staging_folder_delete_no_permission(self):
         self.login_user()

mayan/apps/task_manager/apps.py

@@ -19,6 +19,7 @@ from .links import (
 class TaskManagerApp(MayanAppConfig):
     app_namespace = 'task_manager'
     app_url = 'task_manager'
+    has_tests = True
     name = 'task_manager'
     verbose_name = _('Task manager')
 

mayan/apps/task_manager/tests/literals.py

@@ -0,0 +1,6 @@
+from __future__ import unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+TEST_QUEUE_LABEL = _('Test queue')
+TEST_QUEUE_NAME = 'test_queue'

mayan/apps/task_manager/tests/test_views.py

@@ -0,0 +1,105 @@
+from __future__ import unicode_literals
+
+from common.tests.test_views import GenericViewTestCase
+
+from ..classes import CeleryQueue
+from ..permissions import permission_task_view
+
+from .literals import TEST_QUEUE_LABEL, TEST_QUEUE_NAME
+
+
+class TaskManagerViewTestCase(GenericViewTestCase):
+    def setUp(self):
+        super(TaskManagerViewTestCase, self).setUp()
+        self.test_queue = CeleryQueue(
+            label=TEST_QUEUE_LABEL, name=TEST_QUEUE_NAME
+        )
+
+    def _request_active_task_list(self):
+        return self.get(
+            viewname='task_manager:queue_active_task_list',
+            args=(self.test_queue.name,), follow=True
+        )
+
+    def _request_queue_list(self):
+        return self.get(
+            viewname='task_manager:queue_list', follow=True
+        )
+
+    def _request_reserved_task_list(self):
+        return self.get(
+            viewname='task_manager:queue_reserved_task_list',
+            args=(self.test_queue.name,), follow=True
+        )
+
+    def _request_scheduled_task_list(self):
+        return self.get(
+            viewname='task_manager:queue_scheduled_task_list',
+            args=(self.test_queue.name,), follow=True
+        )
+
+    def test_queue_list_view_no_permissions(self):
+        self.login_user()
+
+        response = self._request_queue_list()
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_queue_list_view_with_permissions(self):
+        self.login_user()
+
+        self.grant_permission(permission=permission_task_view)
+
+        response = self._request_queue_list()
+
+        self.assertContains(
+            response, text=self.test_queue.name, status_code=200
+        )
+
+    def test_active_task_list_view_no_permissions(self):
+        self.login_user()
+
+        response = self._request_active_task_list()
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_active_task_list_view_with_permissions(self):
+        self.login_user()
+
+        self.grant_permission(permission=permission_task_view)
+
+        response = self._request_active_task_list()
+
+        self.assertEqual(response.status_code, 200)
+
+    def test_reserved_task_list_view_no_permissions(self):
+        self.login_user()
+
+        response = self._request_reserved_task_list()
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_reserved_task_list_view_with_permissions(self):
+        self.login_user()
+
+        self.grant_permission(permission=permission_task_view)
+
+        response = self._request_reserved_task_list()
+
+        self.assertEqual(response.status_code, 200)
+
+    def test_scheduled_task_list_view_no_permissions(self):
+        self.login_user()
+
+        response = self._request_scheduled_task_list()
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_scheduled_task_list_view_with_permissions(self):
+        self.login_user()
+
+        self.grant_permission(permission=permission_task_view)
+
+        response = self._request_scheduled_task_list()
+
+        self.assertEqual(response.status_code, 200)

mayan/apps/task_manager/views.py

@@ -16,7 +16,7 @@ class QueueListView(SingleObjectListView):
     }
     view_permission = permission_task_view
 
-    def get_queryset(self):
+    def get_object_list(self):
         return CeleryQueue.all()
 
 
@@ -33,10 +33,7 @@ class QueueActiveTaskListView(SingleObjectListView):
     def get_object(self):
         return CeleryQueue.get(queue_name=self.kwargs['queue_name'])
 
-    def get_task_list(self):
+    def get_object_list(self):
-        return self.get_object().get_active_tasks()
-
-    def get_queryset(self):
         try:
             return self.get_task_list()
         except Exception as exception:
@@ -46,6 +43,9 @@ class QueueActiveTaskListView(SingleObjectListView):
             )
             return ()
 
+    def get_task_list(self):
+        return self.get_object().get_active_tasks()
+
 
 class QueueScheduledTaskListView(QueueActiveTaskListView):
     def get_extra_context(self):

mayan/settings/base.py

@@ -73,6 +73,7 @@ INSTALLED_APPS = (
     'converter',
     'django_gpg',
     'dynamic_search',
+    'kaze',
     'lock_manager',
     'mimetype',
     'navigation',

requirements/base.txt

@@ -39,6 +39,8 @@ python-gnupg==0.3.9
 python-magic==0.4.13
 pytz==2016.7
 
-requests==2.18.4 
+requests==2.18.4
 
 sh==1.12.11
+
+tornado==4.3

serve.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+from django.core.wsgi import get_wsgi_application
+
+import sh
+
+import tornado.httpserver
+import tornado.ioloop
+from tornado.process import Subprocess
+import tornado.web
+import tornado.wsgi
+
+DEFAULT_PORT = 8080
+
+command_manage = sh.Command('./manage.py')
+
+"""
+def add_arguments(self, parser):
+    parser.add_argument(
+        '--single-process',
+        action='store_true',
+        dest='single-process',
+        default=False,
+        help='Forces only one server process.'
+    )
+
+    parser.add_argument(
+        '--port',
+        action='store',
+        dest='port',
+        default=DEFAULT_PORT,
+        help='Port on which to bind the server.'
+    )
+"""
+
+from multiprocessing import Value
+
+from tornado import gen
+from tornado.ioloop import IOLoop
+from tornado.locks import Lock, Semaphore
+
+from tornado.queues import LifoQueue, QueueEmpty
+
+lifo = LifoQueue()
+lock = Lock()
+
+workers = [
+    {
+        'name': 'fast',
+        'queues': 'converter',
+        'concurrency': None,
+    },
+    {
+        'name': 'medium',
+        'queues': 'checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents',
+        'concurrency': None,
+    },
+    {
+        'name': 'slow',
+        'queues': 'mailing,parsing,ocr,tools,statistics',
+        'concurrency': '1',
+        'nice': '19',
+    },
+]
+
+#for worker in workers:
+#    worker_semaphore = Semaphore(1)
+
+#worker_instances = {}
+
+processes = []
+
+
+#@gen.coroutine
+def launch_celery_workers():
+
+    for worker in workers:
+        args = []
+
+        args.extend(['celery', 'worker', '-O', 'fair', '-l', 'INFO'])
+
+        args.extend(['-n', 'mayan-worker-{}.%%h'.format(worker['name'])])
+
+        if 'queues' in worker:
+            args.extend(['-Q', worker['queues']])
+
+        if worker.get('concurrency'):
+            args.extend(['--concurrency', worker.get('concurrency')])
+
+        print 'arguments: ', args
+
+        #with (yield lock.acquire()):
+            #try:
+            #    worker_instances = lifo.get_nowait()
+            #except QueueEmpty:
+            #    worker_instances = {}
+
+        #print 'worker_instances', worker_instances
+
+        #worker_instances.setdefault(worker['name'], 0)
+        #worker_instances[worker['name']] += 1
+
+        #lifo.put(worker_instances)
+
+        #print worker_instances[worker['name']]
+
+        #processes.append(command_nice('-n', worker.get('nice', 0), command_manage(args, _bg=True)))
+        processes.append(command_manage(args, _bg=True))
+        #processes.append(
+        #    subprocess.Popen(
+        #        args, close_fds=True, stderr=subprocess.PIPE,
+        #        stdout=subprocess.PIPE
+        #    )
+        #)
+
+        #Subprocess(args)#, io_loop=global_ioloop)
+
+
+class MayanTornado(object):
+    #@gen.coroutine
+    def start(self):
+        print 'a'
+        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings.production')
+        print '2'
+        option_port = DEFAULT_PORT
+        #option_single_process = False
+        option_fork = 3
+
+        wsgi_application = get_wsgi_application()
+        wsgi_container = tornado.wsgi.WSGIContainer(wsgi_application)
+
+        tornado_application = tornado.web.Application(
+            handlers=(
+                (
+                    r'/static/(.*)', tornado.web.StaticFileHandler,
+                    {'path': 'mayan/media/static'},
+                ),
+                (
+                    '.*', tornado.web.FallbackHandler,
+                    dict(fallback=wsgi_container)
+                ),
+            )
+        )
+
+        http_server = tornado.httpserver.HTTPServer(tornado_application)
+
+        try:
+            if option_fork == 1:
+                http_server.listen(option_port)
+                #ioloop = tornado.ioloop.IOLoop.instance()
+                ioloop = tornado.ioloop.IOLoop.current()
+
+                launch_workers()
+
+                ioloop.start()
+            elif option_fork == 2:
+                http_server.bind(option_port)
+                http_server.start(0)  # forks one process per cpu
+                #global_ioloop = tornado.ioloop.IOLoop.instance()
+                #ioloop = tornado.ioloop.IOLoop.current()
+
+                launch_workers()
+
+                #Subprocess(['./manage.py', 'celery', 'worker', '-O', 'fair', '-l', 'INFO'])
+                #ioloop.start()
+                #global_ioloop.start()
+                tornado.ioloop.IOLoop.current().start()
+            else:
+                launch_celery_workers()
+                sockets = tornado.netutil.bind_sockets(option_port)
+                tornado.process.fork_processes(0)
+
+                #print tornado.ioloop.IOLoop.instance()
+
+                #server = HTTPServer(app)
+                #server.add_sockets(sockets)
+                http_server.add_sockets(sockets)
+                tornado.ioloop.IOLoop.current().start()
+
+        except KeyboardInterrupt:
+            tornado.ioloop.IOLoop.instance().stop()
+            for process in processes:
+                try:
+                    process.process.kill_group()
+                except TypeError:
+                    pass
+
+
+if __name__ == '__main__':
+    app = MayanTornado()
+    app.start()