Merge branch 'master' into feature/tornado

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -360,7 +360,6 @@
 - Add roadmap documentation chapter.
 - API updates.
 
-
 2.0.2 (2016-02-09)
 ==================
 - Install testing dependencies when installing development dependencies.

docs/releases/2.1.rst

@@ -108,6 +108,15 @@ screen. These messages can have an activation and an expiration date and
 time. These messages are useful for display company access policies,
 maintenance announcement, etc.
 
+New server app using tornado
+----------------------------
+This release includes a simple app that can serve Mayan EDMS using the tornado
+server. Using this app users can start using Mayan EDMS with minimal setup
+(just install Redis). By default the server will run on port 52723, but users
+can change the port with the --port option. For privileged port (ports
+below 1024) the command must be run as superadmin. This is an experimental
+feature, feedback and patches are appreciated.
+
 Document signing
 ----------------
 The biggest change for this release if the addition of document signing from

mayan/apps/kaze/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'kaze.apps.KazeApp'

mayan/apps/kaze/apps.py

@@ -0,0 +1,13 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from common import MayanAppConfig
+
+
+class KazeApp(MayanAppConfig):
+    name = 'kaze'
+    verbose_name = _('Kaze')
+
+    def ready(self):
+        super(KazeApp, self).ready()

mayan/apps/kaze/management/commands/serve.py

@@ -0,0 +1,69 @@
+from __future__ import unicode_literals
+
+import os
+
+from django.core import management
+from django.core.wsgi import get_wsgi_application
+
+import tornado.httpserver
+import tornado.ioloop
+from tornado.process import Subprocess
+import tornado.web
+import tornado.wsgi
+
+DEFAULT_PORT = 8080
+
+
+class Command(management.BaseCommand):
+    help = 'Launches a local Tornado server.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--single-process',
+            action='store_true',
+            dest='single-process',
+            default=False,
+            help='Forces only one server process.'
+        )
+
+        parser.add_argument(
+            '--port',
+            action='store',
+            dest='port',
+            default=DEFAULT_PORT,
+            help='Port on which to bind the server.'
+        )
+
+    def handle(self, *args, **options):
+        wsgi_application = get_wsgi_application()
+        wsgi_container = tornado.wsgi.WSGIContainer(wsgi_application)
+
+        tornado_application = tornado.web.Application(
+            handlers=(
+                (
+                    r'/static/(.*)', tornado.web.StaticFileHandler,
+                    {'path': 'mayan/media/static'},
+                ),
+                (
+                    '.*', tornado.web.FallbackHandler,
+                    dict(fallback=wsgi_container)
+                ),
+            )
+        )
+
+        http_server = tornado.httpserver.HTTPServer(tornado_application)
+
+        try:
+            if options['single-process']:
+                http_server.listen(options['port'])
+                ioloop = tornado.ioloop.IOLoop.instance()
+                Subprocess(['./manage.py', 'celery', 'worker', '-O', 'fair'])
+                ioloop.start()
+            else:
+                http_server.bind(options['port'])
+                http_server.start(0)  # forks one process per cpu
+                ioloop = tornado.ioloop.IOLoop.current()
+                Subprocess(['./manage.py', 'celery', 'worker', '-O', 'fair'])
+                ioloop.start()
+        except KeyboardInterrupt:
+            tornado.ioloop.IOLoop.instance().stop()

mayan/settings/base.py

@@ -73,6 +73,7 @@ INSTALLED_APPS = (
     'converter',
     'django_gpg',
     'dynamic_search',
+    'kaze',
     'lock_manager',
     'mimetype',
     'navigation',

requirements/base.txt

@@ -39,6 +39,8 @@ python-gnupg==0.3.9
 python-magic==0.4.13
 pytz==2016.7
 
-requests==2.18.4 
+requests==2.18.4
 
 sh==1.12.11
+
+tornado==4.3

serve.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+from django.core.wsgi import get_wsgi_application
+
+import sh
+
+import tornado.httpserver
+import tornado.ioloop
+from tornado.process import Subprocess
+import tornado.web
+import tornado.wsgi
+
+DEFAULT_PORT = 8080
+
+command_manage = sh.Command('./manage.py')
+
+"""
+def add_arguments(self, parser):
+    parser.add_argument(
+        '--single-process',
+        action='store_true',
+        dest='single-process',
+        default=False,
+        help='Forces only one server process.'
+    )
+
+    parser.add_argument(
+        '--port',
+        action='store',
+        dest='port',
+        default=DEFAULT_PORT,
+        help='Port on which to bind the server.'
+    )
+"""
+
+from multiprocessing import Value
+
+from tornado import gen
+from tornado.ioloop import IOLoop
+from tornado.locks import Lock, Semaphore
+
+from tornado.queues import LifoQueue, QueueEmpty
+
+lifo = LifoQueue()
+lock = Lock()
+
+workers = [
+    {
+        'name': 'fast',
+        'queues': 'converter',
+        'concurrency': None,
+    },
+    {
+        'name': 'medium',
+        'queues': 'checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents',
+        'concurrency': None,
+    },
+    {
+        'name': 'slow',
+        'queues': 'mailing,parsing,ocr,tools,statistics',
+        'concurrency': '1',
+        'nice': '19',
+    },
+]
+
+#for worker in workers:
+#    worker_semaphore = Semaphore(1)
+
+#worker_instances = {}
+
+processes = []
+
+
+#@gen.coroutine
+def launch_celery_workers():
+
+    for worker in workers:
+        args = []
+
+        args.extend(['celery', 'worker', '-O', 'fair', '-l', 'INFO'])
+
+        args.extend(['-n', 'mayan-worker-{}.%%h'.format(worker['name'])])
+
+        if 'queues' in worker:
+            args.extend(['-Q', worker['queues']])
+
+        if worker.get('concurrency'):
+            args.extend(['--concurrency', worker.get('concurrency')])
+
+        print 'arguments: ', args
+
+        #with (yield lock.acquire()):
+            #try:
+            #    worker_instances = lifo.get_nowait()
+            #except QueueEmpty:
+            #    worker_instances = {}
+
+        #print 'worker_instances', worker_instances
+
+        #worker_instances.setdefault(worker['name'], 0)
+        #worker_instances[worker['name']] += 1
+
+        #lifo.put(worker_instances)
+
+        #print worker_instances[worker['name']]
+
+        #processes.append(command_nice('-n', worker.get('nice', 0), command_manage(args, _bg=True)))
+        processes.append(command_manage(args, _bg=True))
+        #processes.append(
+        #    subprocess.Popen(
+        #        args, close_fds=True, stderr=subprocess.PIPE,
+        #        stdout=subprocess.PIPE
+        #    )
+        #)
+
+        #Subprocess(args)#, io_loop=global_ioloop)
+
+
+class MayanTornado(object):
+    #@gen.coroutine
+    def start(self):
+        print 'a'
+        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings.production')
+        print '2'
+        option_port = DEFAULT_PORT
+        #option_single_process = False
+        option_fork = 3
+
+        wsgi_application = get_wsgi_application()
+        wsgi_container = tornado.wsgi.WSGIContainer(wsgi_application)
+
+        tornado_application = tornado.web.Application(
+            handlers=(
+                (
+                    r'/static/(.*)', tornado.web.StaticFileHandler,
+                    {'path': 'mayan/media/static'},
+                ),
+                (
+                    '.*', tornado.web.FallbackHandler,
+                    dict(fallback=wsgi_container)
+                ),
+            )
+        )
+
+        http_server = tornado.httpserver.HTTPServer(tornado_application)
+
+        try:
+            if option_fork == 1:
+                http_server.listen(option_port)
+                #ioloop = tornado.ioloop.IOLoop.instance()
+                ioloop = tornado.ioloop.IOLoop.current()
+
+                launch_workers()
+
+                ioloop.start()
+            elif option_fork == 2:
+                http_server.bind(option_port)
+                http_server.start(0)  # forks one process per cpu
+                #global_ioloop = tornado.ioloop.IOLoop.instance()
+                #ioloop = tornado.ioloop.IOLoop.current()
+
+                launch_workers()
+
+                #Subprocess(['./manage.py', 'celery', 'worker', '-O', 'fair', '-l', 'INFO'])
+                #ioloop.start()
+                #global_ioloop.start()
+                tornado.ioloop.IOLoop.current().start()
+            else:
+                launch_celery_workers()
+                sockets = tornado.netutil.bind_sockets(option_port)
+                tornado.process.fork_processes(0)
+
+                #print tornado.ioloop.IOLoop.instance()
+
+                #server = HTTPServer(app)
+                #server.add_sockets(sockets)
+                http_server.add_sockets(sockets)
+                tornado.ioloop.IOLoop.current().start()
+
+        except KeyboardInterrupt:
+            tornado.ioloop.IOLoop.instance().stop()
+            for process in processes:
+                try:
+                    process.process.kill_group()
+                except TypeError:
+                    pass
+
+
+if __name__ == '__main__':
+    app = MayanTornado()
+    app.start()