diff --git a/README.md b/README.md index 2e9f754566..662d33ab2a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Open source, Django based document manager with custom metadata indexing, file s [Website](http://bit.ly/mayan-edms) -Requirements +Basic requirements --- Python: @@ -15,6 +15,21 @@ Python: * django-filetransfers - File upload/download abstraction * celery- asynchronous task queue/job queue based on distributed message passing * django-celery - celery Django integration +* django-mptt - Utilities for implementing a modified pre-order traversal tree in django +* python-magic - A python wrapper for libmagic +* django-taggit - Simple tagging for django +* slate - The simplest way to extract text from PDFs in Python + + +Execute pip install -r requirements/production.txt to install the python/django dependencies automatically. + +Executables: + +* tesseract-ocr - An OCR Engine that was developed at HP Labs between 1985 and 1995... and now at Google. +* unpaper - post-processing scanned and photocopied book pages + +Optional requirements +--- For the GridFS storage backend: @@ -22,13 +37,12 @@ For the GridFS storage backend: * GridFS - a storage specification for large objects in MongoDB * MongoDB - a scalable, open source, document-oriented database -Or execute pip install -r requirements/production.txt to install the dependencies automatically. +Libraries: -Executables: +* libmagic - MIME detection library, if not installed Mayan will fall back to using python's simpler mimetype built in library + +Mayan has the ability to switch between different image conversion backends, at the moment these two are supported: -* libmagic - MIME detection library -* tesseract-ocr - An OCR Engine that was developed at HP Labs between 1985 and 1995... and now at Google. -* unpaper - post-processing scanned and photocopied book pages * ImageMagick - Convert, Edit, Or Compose Bitmap Images * GraphicMagick - Robust collection of tools and libraries to read, write, and manipulate an image. diff --git a/apps/common/__init__.py b/apps/common/__init__.py index 20657c8419..5bd1d240b6 100644 --- a/apps/common/__init__.py +++ b/apps/common/__init__.py @@ -8,9 +8,7 @@ from django.db.models import signals from navigation.api import register_links from common.conf import settings as common_settings - -TEMPORARY_DIRECTORY = common_settings.TEMPORARY_DIRECTORY \ - if common_settings.TEMPORARY_DIRECTORY else tempfile.mkdtemp() +from common.utils import validate_path def has_usable_password(context): @@ -22,7 +20,6 @@ current_user_edit = {'text': _(u'edit details'), 'view': 'current_user_edit', 'f register_links(['current_user_details', 'current_user_edit', 'password_change_view'], [current_user_details, current_user_edit, password_change_view], menu_name='secondary_menu') - if common_settings.AUTO_CREATE_ADMIN: # From https://github.com/lambdalisue/django-qwert/blob/master/qwert/autoscript/__init__.py # From http://stackoverflow.com/questions/1466827/ -- @@ -50,3 +47,6 @@ if common_settings.AUTO_CREATE_ADMIN: dispatch_uid='django.contrib.auth.management.create_superuser') signals.post_syncdb.connect(create_testuser, sender=auth_models, dispatch_uid='common.models.create_testuser') + +if (validate_path(common_settings.TEMPORARY_DIRECTORY) == False) or (not common_settings.TEMPORARY_DIRECTORY): + setattr(common_settings, 'TEMPORARY_DIRECTORY', tempfile.mkdtemp()) diff --git a/apps/common/templates/generic_list_subtemplate.html b/apps/common/templates/generic_list_subtemplate.html index 2ac52b2dd7..29213c1d97 100644 --- a/apps/common/templates/generic_list_subtemplate.html +++ b/apps/common/templates/generic_list_subtemplate.html @@ -3,6 +3,7 @@ {% load pagination_tags %} {% load navigation_tags %} {% load non_breakable %} +{% load variable_tags %} {% if side_bar %}
@@ -122,13 +123,17 @@ {% endif %} {% endfor %} {% if not hide_links %} + {% if list_object_variable_name %} + {% copy_variable object as list_object_variable_name %} + {% copy_variable list_object_variable_name as "navigation_object_name" %} + {% endif %} {% if navigation_object_links %} {% with navigation_object_links as overrided_object_links %} {% object_navigation_template %} {% endwith %} {% else %} - {% object_navigation_template %} + {% object_navigation_template %} {% endif %} {% endif %} diff --git a/apps/common/templatetags/variable_tags.py b/apps/common/templatetags/variable_tags.py new file mode 100644 index 0000000000..b483834c3a --- /dev/null +++ b/apps/common/templatetags/variable_tags.py @@ -0,0 +1,42 @@ +import re + +from django.template import Node, TemplateSyntaxError, Library, Variable + +register = Library() + + +class CopyNode(Node): + def __init__(self, source_variable, var_name, delete_old=False): + self.source_variable = source_variable + self.var_name = var_name + self.delete_old = delete_old + + def render(self, context): + context[Variable(self.var_name).resolve(context)] = Variable(self.source_variable).resolve(context) + if self.delete_old: + context[Variable(self.source_variable).resolve(context)] = u'' + return '' + + +@register.tag +def copy_variable(parser, token): + return parse_tag(parser, token) + + +@register.tag +def rename_variable(parser, token): + return parse_tag(parser, token, {'delete_old': True}) + + +def parse_tag(parser, token, *args, **kwargs): + # This version uses a regular expression to parse tag contents. + try: + # Splitting by None == splitting by spaces. + tag_name, arg = token.contents.split(None, 1) + except ValueError: + raise TemplateSyntaxError('%r tag requires arguments' % token.contents.split()[0]) + m = re.search(r'(.*?) as ([\'"]*\w+[\'"]*)', arg) + if not m: + raise TemplateSyntaxError('%r tag had invalid arguments' % tag_name) + source_variable, var_name = m.groups() + return CopyNode(source_variable, var_name, *args, **kwargs) diff --git a/apps/common/utils.py b/apps/common/utils.py index 13abb05627..eacaba7923 100644 --- a/apps/common/utils.py +++ b/apps/common/utils.py @@ -2,6 +2,7 @@ import os import re import types +import tempfile from django.utils.http import urlquote as django_urlquote from django.utils.http import urlencode as django_urlencode @@ -12,6 +13,15 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.auth.models import User +try: + from python_magic import magic + USE_PYTHON_MAGIC = True +except: + import mimetypes + mimetypes.init() + USE_PYTHON_MAGIC = False + + def urlquote(link=None, get=None): u''' This method does both: urlquote() and urlencode() @@ -337,3 +347,50 @@ def return_diff(old_obj, new_obj, attrib_list=None): } return diff_dict + + +def get_mimetype(filepath): + """ + Determine a file's mimetype by calling the system's libmagic + library via python-magic or fallback to use python's mimetypes + library + """ + file_mimetype = u'' + file_mime_encoding = u'' + + if USE_PYTHON_MAGIC: + if os.path.exists(filepath): + try: + source = open(filepath, 'r') + mime = magic.Magic(mime=True) + file_mimetype = mime.from_buffer(source.read()) + source.seek(0) + mime_encoding = magic.Magic(mime_encoding=True) + file_mime_encoding = mime_encoding.from_buffer(source.read()) + finally: + if source: + source.close() + else: + path, filename = os.path.split(filepath) + file_mimetype, file_mime_encoding = mimetypes.guess_type(filename) + + return file_mimetype, file_mime_encoding + + +def validate_path(path): + if os.path.exists(path) != True: + # If doesn't exist try to create it + try: + os.mkdir(path) + except: + return False + + # Check if it is writable + try: + fd, test_filepath = tempfile.mkstemp(dir=path) + os.close(fd) + os.unlink(test_filepath) + except: + return False + + return True diff --git a/apps/converter/__init__.py b/apps/converter/__init__.py index 331738373a..ffaef00c09 100644 --- a/apps/converter/__init__.py +++ b/apps/converter/__init__.py @@ -1,11 +1,16 @@ from django.utils.translation import ugettext_lazy as _ +from django.core.exceptions import ImproperlyConfigured from navigation.api import register_sidebar_template -TRANFORMATION_CHOICES = { - u'rotate': u'-rotate %(degrees)d' -} +from converter.utils import load_backend +from converter.conf.settings import GRAPHICS_BACKEND formats_list = {'text': _('file formats'), 'view': 'formats_list', 'famfam': 'pictures'} register_sidebar_template(['formats_list'], 'converter_file_formats_help.html') + +try: + backend = load_backend().ConverterClass() +except ImproperlyConfigured: + raise ImproperlyConfigured(u'Missing or incorrect converter backend: %s' % GRAPHICS_BACKEND) diff --git a/apps/converter/api.py b/apps/converter/api.py index d7595de8c3..3a5b855ada 100644 --- a/apps/converter/api.py +++ b/apps/converter/api.py @@ -1,91 +1,29 @@ import os import subprocess +import hashlib -from django.utils.importlib import import_module -from django.template.defaultfilters import slugify +from common.conf.settings import TEMPORARY_DIRECTORY -from converter.conf.settings import UNPAPER_PATH -from converter.conf.settings import OCR_OPTIONS -from converter.conf.settings import DEFAULT_OPTIONS -from converter.conf.settings import LOW_QUALITY_OPTIONS -from converter.conf.settings import HIGH_QUALITY_OPTIONS -from converter.conf.settings import PRINT_QUALITY_OPTIONS -from converter.conf.settings import GRAPHICS_BACKEND from converter.conf.settings import UNOCONV_PATH +from converter.exceptions import OfficeConversionError +from converter.literals import DEFAULT_PAGE_NUMBER, \ + DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, DEFAULT_FILE_FORMAT -from converter.exceptions import UnpaperError, OfficeConversionError - -from common import TEMPORARY_DIRECTORY -from documents.utils import document_save_to_temp_dir - -DEFAULT_ZOOM_LEVEL = 100 -DEFAULT_ROTATION = 0 -DEFAULT_PAGE_INDEX_NUMBER = 0 -DEFAULT_FILE_FORMAT = u'jpg' -DEFAULT_OCR_FILE_FORMAT = u'tif' - -QUALITY_DEFAULT = u'quality_default' -QUALITY_LOW = u'quality_low' -QUALITY_HIGH = u'quality_high' -QUALITY_PRINT = u'quality_print' - -QUALITY_SETTINGS = { - QUALITY_DEFAULT: DEFAULT_OPTIONS, - QUALITY_LOW: LOW_QUALITY_OPTIONS, - QUALITY_HIGH: HIGH_QUALITY_OPTIONS, - QUALITY_PRINT: PRINT_QUALITY_OPTIONS -} +from converter import backend +from converter.literals import TRANSFORMATION_CHOICES +from converter.literals import TRANSFORMATION_RESIZE, \ + TRANSFORMATION_ROTATE, TRANSFORMATION_DENSITY, \ + TRANSFORMATION_ZOOM +from converter.literals import DIMENSION_SEPARATOR +from converter.utils import cleanup +HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() + CONVERTER_OFFICE_FILE_EXTENSIONS = [ u'ods', u'docx', u'doc' ] -def _lazy_load(fn): - _cached = [] - - def _decorated(): - if not _cached: - _cached.append(fn()) - return _cached[0] - return _decorated - - -@_lazy_load -def _get_backend(): - return import_module(GRAPHICS_BACKEND) - -try: - backend = _get_backend() -except ImportError: - raise ImportError(u'Missing or incorrect converter backend: %s' % GRAPHICS_BACKEND) - - -def cleanup(filename): - """ - Tries to remove the given filename. Ignores non-existent files - """ - try: - os.remove(filename) - except OSError: - pass - - -def execute_unpaper(input_filepath, output_filepath): - """ - Executes the program unpaper using subprocess's Popen - """ - command = [] - command.append(UNPAPER_PATH) - command.append(u'--overwrite') - command.append(input_filepath) - command.append(output_filepath) - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - raise UnpaperError(proc.stderr.readline()) - - def execute_unoconv(input_filepath, arguments=''): """ Executes the program unoconv using subprocess's Popen @@ -109,19 +47,11 @@ def cache_cleanup(input_filepath, *args, **kwargs): def create_image_cache_filename(input_filepath, *args, **kwargs): if input_filepath: - temp_filename, separator = os.path.splitext(os.path.basename(input_filepath)) - temp_path = os.path.join(TEMPORARY_DIRECTORY, temp_filename) - - final_filepath = [] - [final_filepath.append(str(arg)) for arg in args] - final_filepath.extend([u'%s_%s' % (key, value) for key, value in kwargs.items()]) - - temp_path += slugify(u'_'.join(final_filepath)) - - return temp_path + hash_value = HASH_FUNCTION(u''.join([input_filepath, unicode(args), unicode(kwargs)])) + return os.path.join(TEMPORARY_DIRECTORY, hash_value) else: return None - + def convert_office_document(input_filepath): if os.path.exists(UNOCONV_PATH): @@ -130,27 +60,19 @@ def convert_office_document(input_filepath): return None -def convert_document(document, *args, **kwargs): - document_filepath = create_image_cache_filename(document.checksum, *args, **kwargs) - if os.path.exists(document_filepath): - return document_filepath - - return convert(document_save_to_temp_dir(document, document.checksum), *args, **kwargs) - - -def convert(input_filepath, *args, **kwargs): +def convert(input_filepath, output_filepath=None, cleanup_files=False, *args, **kwargs): size = kwargs.get('size') file_format = kwargs.get('file_format', DEFAULT_FILE_FORMAT) - extra_options = kwargs.get('extra_options', u'') zoom = kwargs.get('zoom', DEFAULT_ZOOM_LEVEL) rotation = kwargs.get('rotation', DEFAULT_ROTATION) - page = kwargs.get('page', DEFAULT_PAGE_INDEX_NUMBER) - cleanup_files = kwargs.get('cleanup_files', True) - quality = kwargs.get('quality', QUALITY_DEFAULT) + page = kwargs.get('page', DEFAULT_PAGE_NUMBER) + transformations = kwargs.get('transformations', []) unoconv_output = None - output_filepath = create_image_cache_filename(input_filepath, *args, **kwargs) + if output_filepath is None: + output_filepath = create_image_cache_filename(input_filepath, *args, **kwargs) + if os.path.exists(output_filepath): return output_filepath @@ -160,20 +82,33 @@ def convert(input_filepath, *args, **kwargs): if result: unoconv_output = result input_filepath = result - extra_options = u'' - input_arg = u'%s[%s]' % (input_filepath, page) - extra_options += u' -resize %s' % size + if size: + transformations.append( + { + 'transformation': TRANSFORMATION_RESIZE, + 'arguments': dict(zip([u'width', u'height'], size.split(DIMENSION_SEPARATOR))) + } + ) + if zoom != 100: - extra_options += u' -resize %d%% ' % zoom + transformations.append( + { + 'transformation': TRANSFORMATION_ZOOM, + 'arguments': {'percent': zoom} + } + ) if rotation != 0 and rotation != 360: - extra_options += u' -rotate %d ' % rotation + transformations.append( + { + 'transformation': TRANSFORMATION_ROTATE, + 'arguments': {'degrees': rotation} + } + ) - if format == u'jpg': - extra_options += u' -quality 85' try: - backend.execute_convert(input_filepath=input_arg, arguments=extra_options, output_filepath=u'%s:%s' % (file_format, output_filepath), quality=quality) + backend.convert_file(input_filepath=input_filepath, output_filepath=output_filepath, transformations=transformations, page=page, file_format=file_format) finally: if cleanup_files: cleanup(input_filepath) @@ -184,51 +119,22 @@ def convert(input_filepath, *args, **kwargs): def get_page_count(input_filepath): - try: - return len(backend.execute_identify(unicode(input_filepath)).splitlines()) - except: - #TODO: send to other page number identifying program - return 1 + return backend.get_page_count(input_filepath) def get_document_dimensions(document, *args, **kwargs): document_filepath = create_image_cache_filename(document.checksum, *args, **kwargs) if os.path.exists(document_filepath): options = [u'-format', u'%w %h'] - return [int(dimension) for dimension in backend.execute_identify(unicode(document_filepath), options).split()] + return [int(dimension) for dimension in backend.identify_file(unicode(document_filepath), options).split()] else: return [0, 0] -def convert_document_for_ocr(document, page=DEFAULT_PAGE_INDEX_NUMBER, file_format=DEFAULT_OCR_FILE_FORMAT): - #Extract document file - input_filepath = document_save_to_temp_dir(document, document.uuid) - - #Convert for OCR - temp_filename, separator = os.path.splitext(os.path.basename(input_filepath)) - temp_path = os.path.join(TEMPORARY_DIRECTORY, temp_filename) - transformation_output_file = u'%s_trans%s%s%s' % (temp_path, page, os.extsep, file_format) - unpaper_input_file = u'%s_unpaper_in%s%spnm' % (temp_path, page, os.extsep) - unpaper_output_file = u'%s_unpaper_out%s%spnm' % (temp_path, page, os.extsep) - convert_output_file = u'%s_ocr%s%s%s' % (temp_path, page, os.extsep, file_format) - - input_arg = u'%s[%s]' % (input_filepath, page) - - try: - document_page = document.documentpage_set.get(page_number=page + 1) - transformation_string, warnings = document_page.get_transformation_string() - - #Apply default transformations - backend.execute_convert(input_filepath=input_arg, quality=QUALITY_HIGH, arguments=transformation_string, output_filepath=transformation_output_file) - #Do OCR operations - backend.execute_convert(input_filepath=transformation_output_file, arguments=OCR_OPTIONS, output_filepath=unpaper_input_file) - # Process by unpaper - execute_unpaper(input_filepath=unpaper_input_file, output_filepath=unpaper_output_file) - # Convert to tif - backend.execute_convert(input_filepath=unpaper_output_file, output_filepath=convert_output_file) - finally: - cleanup(transformation_output_file) - cleanup(unpaper_input_file) - cleanup(unpaper_output_file) - - return convert_output_file +def get_available_transformations_choices(): + result = [] + for transformation in backend.get_available_transformations(): + transformation_template = u'%s %s' % (TRANSFORMATION_CHOICES[transformation]['label'], u','.join(['<%s>' % argument['name'] if argument['required'] else '[%s]' % argument['name'] for argument in TRANSFORMATION_CHOICES[transformation]['arguments']])) + result.append([transformation, transformation_template]) + + return result diff --git a/apps/converter/backends/__init__.py b/apps/converter/backends/__init__.py index e69de29bb2..a98881632b 100644 --- a/apps/converter/backends/__init__.py +++ b/apps/converter/backends/__init__.py @@ -0,0 +1,18 @@ +class ConverterBase(object): + """ + Base class that all backend classes must inherit + """ + def convert_file(self, input_filepath, *args, **kwargs): + raise NotImplementedError("Your %s class has not defined a convert_file() method, which is required." % self.__class__.__name__) + + def convert_document(self, document, *args, **kwargs): + raise NotImplementedError("Your %s class has not defined a convert_document() method, which is required." % self.__class__.__name__) + + def get_format_list(self): + raise NotImplementedError("Your %s class has not defined a get_format_list() method, which is required." % self.__class__.__name__) + + def get_available_transformations(self): + raise NotImplementedError("Your %s class has not defined a get_available_transformations() method, which is required." % self.__class__.__name__) + + def get_page_count(self): + raise NotImplementedError("Your %s class has not defined a get_page_count() method, which is required." % self.__class__.__name__) diff --git a/apps/converter/backends/graphicsmagick.py b/apps/converter/backends/graphicsmagick.py deleted file mode 100644 index 360a24a58b..0000000000 --- a/apps/converter/backends/graphicsmagick.py +++ /dev/null @@ -1,71 +0,0 @@ -import subprocess -import re - -from converter.conf.settings import GM_PATH -from converter.conf.settings import GM_SETTINGS -from converter.api import QUALITY_DEFAULT, QUALITY_SETTINGS -from converter.exceptions import ConvertError, UnknownFormat, IdentifyError - -CONVERTER_ERROR_STRING_NO_DECODER = u'No decode delegate for this image format' -CONVERTER_ERROR_STARTS_WITH = u'starts with' - - -def execute_identify(input_filepath, arguments=None): - command = [] - command.append(unicode(GM_PATH)) - command.append(u'identify') - if arguments: - command.extend(arguments) - command.append(unicode(input_filepath)) - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - raise IdentifyError(proc.stderr.readline()) - return proc.stdout.read() - - -def execute_convert(input_filepath, output_filepath, quality=QUALITY_DEFAULT, arguments=None): - command = [] - command.append(unicode(GM_PATH)) - command.append(u'convert') - command.extend(unicode(QUALITY_SETTINGS[quality]).split()) - command.extend(unicode(GM_SETTINGS).split()) - command.append(unicode(input_filepath)) - if arguments: - command.extend(unicode(arguments).split()) - command.append(unicode(output_filepath)) - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - #Got an error from convert program - error_line = proc.stderr.readline() - if (CONVERTER_ERROR_STRING_NO_DECODER in error_line) or (CONVERTER_ERROR_STARTS_WITH in error_line): - #Try to determine from error message which class of error is it - raise UnknownFormat - else: - raise ConvertError(error_line) - - -def get_format_list(): - """ - Call GraphicsMagick to parse all of it's supported file formats, and - return a list of the names and descriptions - """ - format_regex = re.compile(' *([A-Z0-9]+)[*]? +([A-Z0-9]+) +([rw\-+]+) *(.*).*') - formats = [] - command = [] - command.append(unicode(GM_PATH)) - command.append(u'convert') - command.append(u'-list') - command.append(u'formats') - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - raise ConvertError(proc.stderr.readline()) - - for line in proc.stdout.readlines(): - fields = format_regex.findall(line) - if fields: - formats.append((fields[0][0], fields[0][3])) - - return formats diff --git a/apps/converter/backends/base.py b/apps/converter/backends/graphicsmagick/__init__.py similarity index 100% rename from apps/converter/backends/base.py rename to apps/converter/backends/graphicsmagick/__init__.py diff --git a/apps/converter/backends/graphicsmagick/base.py b/apps/converter/backends/graphicsmagick/base.py new file mode 100644 index 0000000000..1d70108a94 --- /dev/null +++ b/apps/converter/backends/graphicsmagick/base.py @@ -0,0 +1,119 @@ +import subprocess +import re + +from converter.conf.settings import GM_PATH +from converter.conf.settings import GM_SETTINGS +from converter.exceptions import ConvertError, UnknownFormat, \ + IdentifyError +from converter.backends import ConverterBase +from converter.literals import TRANSFORMATION_RESIZE, \ + TRANSFORMATION_ROTATE, TRANSFORMATION_DENSITY, \ + TRANSFORMATION_ZOOM +from converter.literals import DIMENSION_SEPARATOR, DEFAULT_PAGE_NUMBER, \ + DEFAULT_FILE_FORMAT + +CONVERTER_ERROR_STRING_NO_DECODER = u'No decode delegate for this image format' +CONVERTER_ERROR_STARTS_WITH = u'starts with' + + +class ConverterClass(ConverterBase): + def identify_file(self, input_filepath, arguments=None): + command = [] + command.append(unicode(GM_PATH)) + command.append(u'identify') + if arguments: + command.extend(arguments) + command.append(unicode(input_filepath)) + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + raise IdentifyError(proc.stderr.readline()) + return proc.stdout.read() + + def convert_file(self, input_filepath, output_filepath, transformations=None, page=DEFAULT_PAGE_NUMBER, file_format=DEFAULT_FILE_FORMAT): + arguments = [] + + + if transformations: + for transformation in transformations: + if transformation['transformation'] == TRANSFORMATION_RESIZE: + dimensions = [] + dimensions.append(unicode(transformation['arguments']['width'])) + if 'height' in transformation['arguments']: + dimensions.append(unicode(transformation['arguments']['height'])) + arguments.append(u'-resize') + arguments.append(u'%s' % DIMENSION_SEPARATOR.join(dimensions)) + + elif transformation['transformation'] == TRANSFORMATION_ZOOM: + arguments.append(u'-resize') + arguments.append(u'%d%%' % transformation['arguments']['percent']) + + elif transformation['transformation'] == TRANSFORMATION_ROTATE: + arguments.append(u'-rotate') + arguments.append(u'%s' % transformation['arguments']['degrees']) + + if file_format.lower() == u'jpeg' or file_format.lower() == u'jpg': + arguments.append(u'-quality') + arguments.append(u'85') + + # Graphicsmagick page number is 0 base + input_arg = u'%s[%d]' % (input_filepath, page - 1) + + # Specify the file format next to the output filename + output_filepath = u'%s:%s' % (file_format, output_filepath) + + command = [] + command.append(unicode(GM_PATH)) + command.append(u'convert') + command.extend(unicode(GM_SETTINGS).split()) + command.append(unicode(input_arg)) + if arguments: + command.extend(arguments) + command.append(unicode(output_filepath)) + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + #Got an error from convert program + error_line = proc.stderr.readline() + if (CONVERTER_ERROR_STRING_NO_DECODER in error_line) or (CONVERTER_ERROR_STARTS_WITH in error_line): + #Try to determine from error message which class of error is it + raise UnknownFormat + else: + raise ConvertError(error_line) + + def get_format_list(self): + """ + Call GraphicsMagick to parse all of it's supported file formats, and + return a list of the names and descriptions + """ + format_regex = re.compile(' *([A-Z0-9]+)[*]? +([A-Z0-9]+) +([rw\-+]+) *(.*).*') + formats = [] + command = [] + command.append(unicode(GM_PATH)) + command.append(u'convert') + command.append(u'-list') + command.append(u'formats') + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + raise ConvertError(proc.stderr.readline()) + + for line in proc.stdout.readlines(): + fields = format_regex.findall(line) + if fields: + formats.append((fields[0][0], fields[0][3])) + + return formats + + def get_available_transformations(self): + return [ + TRANSFORMATION_RESIZE, TRANSFORMATION_ROTATE, \ + TRANSFORMATION_ZOOM + ] + + def get_page_count(self, input_filepath): + try: + return len(self.identify_file(unicode(input_filepath)).splitlines()) + except: + #TODO: send to other page number identifying program + return 1 diff --git a/apps/converter/backends/imagemagick.py b/apps/converter/backends/imagemagick.py deleted file mode 100644 index 4542ebdeba..0000000000 --- a/apps/converter/backends/imagemagick.py +++ /dev/null @@ -1,68 +0,0 @@ -import subprocess -import re - -from converter.conf.settings import IM_IDENTIFY_PATH -from converter.conf.settings import IM_CONVERT_PATH -from converter.api import QUALITY_DEFAULT, QUALITY_SETTINGS -from converter.exceptions import ConvertError, UnknownFormat, \ - IdentifyError - -CONVERTER_ERROR_STRING_NO_DECODER = u'no decode delegate for this image format' - - -def execute_identify(input_filepath, arguments=None): - command = [] - command.append(unicode(IM_IDENTIFY_PATH)) - if arguments: - command.extend(arguments) - command.append(unicode(input_filepath)) - - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - raise IdentifyError(proc.stderr.readline()) - return proc.stdout.read() - - -def execute_convert(input_filepath, output_filepath, quality=QUALITY_DEFAULT, arguments=None): - command = [] - command.append(unicode(IM_CONVERT_PATH)) - command.extend(unicode(QUALITY_SETTINGS[quality]).split()) - command.append(unicode(input_filepath)) - if arguments: - command.extend(unicode(arguments).split()) - command.append(unicode(output_filepath)) - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - #Got an error from convert program - error_line = proc.stderr.readline() - if CONVERTER_ERROR_STRING_NO_DECODER in error_line: - #Try to determine from error message which class of error is it - raise UnknownFormat - else: - raise ConvertError(error_line) - - -def get_format_list(): - """ - Call ImageMagick to parse all of it's supported file formats, and - return a list of the names and descriptions - """ - format_regex = re.compile(' *([A-Z0-9]+)[*]? +([A-Z0-9]+) +([rw\-+]+) *(.*).*') - formats = [] - command = [] - command.append(unicode(IM_CONVERT_PATH)) - command.append(u'-list') - command.append(u'format') - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - raise ConvertError(proc.stderr.readline()) - - for line in proc.stdout.readlines(): - fields = format_regex.findall(line) - if fields: - formats.append((fields[0][0], fields[0][3])) - - return formats diff --git a/apps/converter/backends/imagemagick/__init__.py b/apps/converter/backends/imagemagick/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/converter/backends/imagemagick/base.py b/apps/converter/backends/imagemagick/base.py new file mode 100644 index 0000000000..977da783c2 --- /dev/null +++ b/apps/converter/backends/imagemagick/base.py @@ -0,0 +1,116 @@ +import subprocess +import re + +from converter.conf.settings import IM_IDENTIFY_PATH +from converter.conf.settings import IM_CONVERT_PATH +from converter.exceptions import ConvertError, UnknownFormat, \ + IdentifyError +from converter.backends import ConverterBase +from converter.literals import TRANSFORMATION_RESIZE, \ + TRANSFORMATION_ROTATE, TRANSFORMATION_DENSITY, \ + TRANSFORMATION_ZOOM +from converter.literals import DIMENSION_SEPARATOR, DEFAULT_PAGE_NUMBER, \ + DEFAULT_FILE_FORMAT + +CONVERTER_ERROR_STRING_NO_DECODER = u'no decode delegate for this image format' + + +class ConverterClass(ConverterBase): + def identify_file(self, input_filepath, arguments=None): + command = [] + command.append(unicode(IM_IDENTIFY_PATH)) + if arguments: + command.extend(arguments) + command.append(unicode(input_filepath)) + + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + raise IdentifyError(proc.stderr.readline()) + return proc.stdout.read() + + def convert_file(self, input_filepath, output_filepath, transformations=None, page=DEFAULT_PAGE_NUMBER, file_format=DEFAULT_FILE_FORMAT): + arguments = [] + if transformations: + for transformation in transformations: + if transformation['transformation'] == TRANSFORMATION_RESIZE: + dimensions = [] + dimensions.append(unicode(transformation['arguments']['width'])) + if 'height' in transformation['arguments']: + dimensions.append(unicode(transformation['arguments']['height'])) + arguments.append(u'-resize') + arguments.append(u'%s' % DIMENSION_SEPARATOR.join(dimensions)) + + elif transformation['transformation'] == TRANSFORMATION_ZOOM: + arguments.append(u'-resize') + arguments.append(u'%d%%' % transformation['arguments']['percent']) + + elif transformation['transformation'] == TRANSFORMATION_ROTATE: + arguments.append(u'-rotate') + arguments.append(u'%s' % transformation['arguments']['degrees']) + + if file_format.lower() == u'jpeg' or file_format.lower() == u'jpg': + arguments.append(u'-quality') + arguments.append(u'85') + + # Imagemagick page number is 0 base + input_arg = u'%s[%d]' % (input_filepath, page - 1) + + # Specify the file format next to the output filename + output_filepath = u'%s:%s' % (file_format, output_filepath) + + command = [] + command.append(unicode(IM_CONVERT_PATH)) + command.append(unicode(input_arg)) + if arguments: + command.extend(arguments) + command.append(unicode(output_filepath)) + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + #Got an error from convert program + error_line = proc.stderr.readline() + if CONVERTER_ERROR_STRING_NO_DECODER in error_line: + #Try to determine from error message which class of error is it + raise UnknownFormat + else: + raise ConvertError(error_line) + + + def get_format_list(self): + """ + Call ImageMagick to parse all of it's supported file formats, and + return a list of the names and descriptions + """ + format_regex = re.compile(' *([A-Z0-9]+)[*]? +([A-Z0-9]+) +([rw\-+]+) *(.*).*') + formats = [] + command = [] + command.append(unicode(IM_CONVERT_PATH)) + command.append(u'-list') + command.append(u'format') + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + raise ConvertError(proc.stderr.readline()) + + for line in proc.stdout.readlines(): + fields = format_regex.findall(line) + if fields: + formats.append((fields[0][0], fields[0][3])) + + return formats + + + def get_available_transformations(self): + return [ + TRANSFORMATION_RESIZE, TRANSFORMATION_ROTATE, \ + TRANSFORMATION_ZOOM + ] + + + def get_page_count(self, input_filepath): + try: + return len(self.identify_file(unicode(input_filepath)).splitlines()) + except: + #TODO: send to other page number identifying program + return 1 diff --git a/apps/converter/backends/python/__init__.py b/apps/converter/backends/python/__init__.py new file mode 100644 index 0000000000..dfeca950f1 --- /dev/null +++ b/apps/converter/backends/python/__init__.py @@ -0,0 +1,3 @@ +from PIL import Image + +Image.init() diff --git a/apps/converter/backends/python/base.py b/apps/converter/backends/python/base.py new file mode 100644 index 0000000000..e854ab6243 --- /dev/null +++ b/apps/converter/backends/python/base.py @@ -0,0 +1,171 @@ +import tempfile +import os + +import slate +from PIL import Image +import ghostscript + +from django.utils.translation import ugettext_lazy as _ + +from common.utils import get_mimetype + +from converter.exceptions import ConvertError, UnknownFormat, IdentifyError +from converter.backends import ConverterBase +from converter.literals import TRANSFORMATION_RESIZE, \ + TRANSFORMATION_ROTATE, TRANSFORMATION_ZOOM +from converter.literals import DEFAULT_PAGE_NUMBER, \ + DEFAULT_FILE_FORMAT +from converter.utils import cleanup + + +class ConverterClass(ConverterBase): + def get_page_count(self, input_filepath): + page_count = 1 + + mimetype, encoding = get_mimetype(input_filepath) + if mimetype == 'application/pdf': + # If file is a PDF open it with slate to determine the page + # count + with open(input_filepath) as fd: + pages = slate.PDF(fd) + return len(pages) + + try: + im = Image.open(input_filepath) + except IOError: #cannot identify image file + # Return a page count of 1, to atleast allow the document + # to be created + return 1 + + try: + while 1: + im.seek(im.tell()+1) + page_count += 1 + # do something to im + except EOFError: + pass # end of sequence + + return page_count + + def convert_file(self, input_filepath, output_filepath, transformations=None, page=DEFAULT_PAGE_NUMBER, file_format=DEFAULT_FILE_FORMAT): + tmpfile = None + mimetype, encoding = get_mimetype(input_filepath) + if mimetype == 'application/pdf': + # If file is a PDF open it with ghostscript and convert it to + # TIFF + first_page_tmpl = '-dFirstPage=%d' % page + last_page_tmpl = '-dLastPage=%d' % page + fd, tmpfile = tempfile.mkstemp() + os.close(fd) + output_file_tmpl = '-sOutputFile=%s' % tmpfile + input_file_tmpl = '-f%s' % input_filepath + args = [ + 'gs', '-q', '-dQUIET', '-dSAFER', '-dBATCH', + '-dNOPAUSE', '-dNOPROMPT', + first_page_tmpl, last_page_tmpl, + '-sDEVICE=jpeg', '-dJPEGQ=75', + '-r150', output_file_tmpl, + input_file_tmpl, + '-c "60000000 setvmthreshold"', # use 30MB + '-dNOGC', # No garbage collection + '-dMaxBitmap=500000000', + '-dAlignToPixels=0', + '-dGridFitTT=0', + '-dTextAlphaBits=4', + '-dGraphicsAlphaBits=4', + ] + + ghostscript.Ghostscript(*args) + page = 1 # Don't execute the following while loop + input_filepath = tmpfile + + try: + im = Image.open(input_filepath) + except Exception: # Python Imaging Library doesn't recognize it as an image + raise UnknownFormat + finally: + if tmpfile: + cleanup(tmpfile) + + current_page = 0 + try: + while current_page == page - 1: + im.seek(im.tell() + 1) + current_page += 1 + # do something to im + except EOFError: + pass # end of sequence + + if transformations: + aspect = 1.0 * im.size[0] / im.size[1] + for transformation in transformations: + if transformation['transformation'] == TRANSFORMATION_RESIZE: + width = int(transformation['arguments']['width']) + height = int(transformation['arguments'].get('height', 1.0 * width * aspect)) + im = self.resize(im, (width, height)) + elif transformation['transformation'] == TRANSFORMATION_ZOOM: + decimal_value = float(transformation['arguments']['percent']) / 100 + im = im.transform((im.size[0] * decimal_value, im.size[1] * decimal_value), Image.EXTENT, (0, 0, im.size[0], im.size[1])) + elif transformation['transformation'] == TRANSFORMATION_ROTATE: + # PIL counter degress counter-clockwise, reverse them + im = im.rotate(360 - transformation['arguments']['degrees']) + + if im.mode not in ('L', 'RGB'): + im = im.convert('RGB') + + im.save(output_filepath, format=file_format) + + def get_format_list(self): + """ + Introspect PIL's internal registry to obtain a list of the + supported file types + """ + formats = [] + for format_name in Image.ID: + formats.append((format_name, u'')) + + return formats + + def get_available_transformations(self): + return [ + TRANSFORMATION_RESIZE, TRANSFORMATION_ROTATE, \ + TRANSFORMATION_ZOOM + ] + + # From: http://united-coders.com/christian-harms/image-resizing-tips-general-and-for-python + def resize(self, img, box, fit=False, out=None): + '''Downsample the image. + @param img: Image - an Image-object + @param box: tuple(x, y) - the bounding box of the result image + @param fit: boolean - crop the image to fill the box + @param out: file-like-object - save the image into the output stream + ''' + #preresize image with factor 2, 4, 8 and fast algorithm + factor = 1 + while img.size[0]/factor > 2*box[0] and img.size[1]*2/factor > 2*box[1]: + factor *=2 + if factor > 1: + img.thumbnail((img.size[0]/factor, img.size[1]/factor), Image.NEAREST) + + #calculate the cropping box and get the cropped part + if fit: + x1 = y1 = 0 + x2, y2 = img.size + wRatio = 1.0 * x2/box[0] + hRatio = 1.0 * y2/box[1] + if hRatio > wRatio: + y1 = y2/2-box[1]*wRatio/2 + y2 = y2/2+box[1]*wRatio/2 + else: + x1 = x2/2-box[0]*hRatio/2 + x2 = x2/2+box[0]*hRatio/2 + img = img.crop((x1,y1,x2,y2)) + + #Resize the image with best quality algorithm ANTI-ALIAS + img.thumbnail(box, Image.ANTIALIAS) + + if out: + #save it into a file-like object + img.save(out, "JPEG", quality=75) + else: + return img diff --git a/apps/converter/conf/settings.py b/apps/converter/conf/settings.py index f73c0f2b64..08377880b4 100644 --- a/apps/converter/conf/settings.py +++ b/apps/converter/conf/settings.py @@ -9,15 +9,12 @@ register_settings( settings=[ {'name': u'IM_CONVERT_PATH', 'global_name': u'CONVERTER_IM_CONVERT_PATH', 'default': u'/usr/bin/convert', 'description': _(u'File path to imagemagick\'s convert program.'), 'exists': True}, {'name': u'IM_IDENTIFY_PATH', 'global_name': u'CONVERTER_IM_IDENTIFY_PATH', 'default': u'/usr/bin/identify', 'description': _(u'File path to imagemagick\'s identify program.'), 'exists': True}, - {'name': u'UNPAPER_PATH', 'global_name': u'CONVERTER_UNPAPER_PATH', 'default': u'/usr/bin/unpaper', 'description': _(u'File path to unpaper program.'), 'exists': True}, {'name': u'GM_PATH', 'global_name': u'CONVERTER_GM_PATH', 'default': u'/usr/bin/gm', 'description': _(u'File path to graphicsmagick\'s program.'), 'exists': True}, {'name': u'GM_SETTINGS', 'global_name': u'CONVERTER_GM_SETTINGS', 'default': u''}, - {'name': u'GRAPHICS_BACKEND', 'global_name': u'CONVERTER_GRAPHICS_BACKEND', 'default': u'converter.backends.imagemagick', 'description': _(u'Graphics conversion backend to use. Options are: converter.backends.imagemagick and converter.backends.graphicsmagick.')}, + {'name': u'GRAPHICS_BACKEND', 'global_name': u'CONVERTER_GRAPHICS_BACKEND', 'default': u'converter.backends.python', 'description': _(u'Graphics conversion backend to use. Options are: converter.backends.imagemagick, converter.backends.graphicsmagick and converter.backends.python.')}, {'name': u'UNOCONV_PATH', 'global_name': u'CONVERTER_UNOCONV_PATH', 'default': u'/usr/bin/unoconv', 'exists': True}, - {'name': u'OCR_OPTIONS', 'global_name': u'CONVERTER_OCR_OPTIONS', 'default': u'-colorspace Gray -depth 8 -resample 200x200'}, - {'name': u'DEFAULT_OPTIONS', 'global_name': u'CONVERTER_DEFAULT_OPTIONS', 'default': u''}, - {'name': u'LOW_QUALITY_OPTIONS', 'global_name': u'CONVERTER_LOW_QUALITY_OPTIONS', 'default': u''}, - {'name': u'HIGH_QUALITY_OPTIONS', 'global_name': u'CONVERTER_HIGH_QUALITY_OPTIONS', 'default': u'-density 400'}, - {'name': u'PRINT_QUALITY_OPTIONS', 'global_name': u'CONVERTER_PRINT_QUALITY_OPTIONS', 'default': u'-density 500'}, + #{'name': u'OCR_OPTIONS', 'global_name': u'CONVERTER_OCR_OPTIONS', 'default': u'-colorspace Gray -depth 8 -resample 200x200'}, + #{'name': u'HIGH_QUALITY_OPTIONS', 'global_name': u'CONVERTER_HIGH_QUALITY_OPTIONS', 'default': u'-density 400'}, + #{'name': u'PRINT_QUALITY_OPTIONS', 'global_name': u'CONVERTER_PRINT_QUALITY_OPTIONS', 'default': u'-density 500'}, ] ) diff --git a/apps/converter/exceptions.py b/apps/converter/exceptions.py index c906fc5c95..1880f0ba39 100644 --- a/apps/converter/exceptions.py +++ b/apps/converter/exceptions.py @@ -13,13 +13,6 @@ class UnknownFormat(ConvertError): pass -class UnpaperError(ConvertError): - """ - Raised by unpaper - """ - pass - - class IdentifyError(ConvertError): """ Raised by identify diff --git a/apps/converter/literals.py b/apps/converter/literals.py new file mode 100644 index 0000000000..66a17d0d67 --- /dev/null +++ b/apps/converter/literals.py @@ -0,0 +1,46 @@ +from django.utils.translation import ugettext_lazy as _ + +DEFAULT_ZOOM_LEVEL = 100 +DEFAULT_ROTATION = 0 +DEFAULT_PAGE_NUMBER = 1 +DEFAULT_FILE_FORMAT = u'jpeg' + +DIMENSION_SEPARATOR = u'x' + +TRANSFORMATION_RESIZE = u'resize' +TRANSFORMATION_ROTATE = u'rotate' +TRANSFORMATION_DENSITY = u'density' +TRANSFORMATION_ZOOM = u'zoom' + +TRANSFORMATION_CHOICES = { + TRANSFORMATION_RESIZE: { + 'label': _(u'Resize'), + 'description': _(u'Resize.'), + 'arguments': [ + {'name': 'width', 'label': _(u'width'), 'required': True}, + {'name': 'height', 'label': _(u'height'), 'required': False}, + ] + }, + TRANSFORMATION_ROTATE: { + 'label': _(u'Rotate'), + 'description': _(u'Rotate by n degress.'), + 'arguments': [ + {'name': 'degrees', 'label': _(u'degrees'), 'required': True} + ] + }, + TRANSFORMATION_DENSITY: { + 'label': _(u'Density'), + 'description': _(u'Change the resolution (ie: DPI) without resizing.'), + 'arguments': [ + {'name': 'width', 'label': _(u'width'), 'required': True}, + {'name': 'height', 'label': _(u'height'), 'required': False}, + ] + }, + TRANSFORMATION_ZOOM: { + 'label': _(u'Zoom'), + 'description': _(u'Zoom by n percent.'), + 'arguments': [ + {'name': 'percent', 'label': _(u'percent'), 'required': True} + ] + }, +} diff --git a/apps/converter/utils.py b/apps/converter/utils.py index c5a4e7e55b..26ad9c4b74 100644 --- a/apps/converter/utils.py +++ b/apps/converter/utils.py @@ -1,6 +1,10 @@ +import os + +from django.core.exceptions import ImproperlyConfigured +from django.utils.importlib import import_module + + #http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python - - def copyfile(source, dest, buffer_size=1024 * 1024): """ Copy a file from source to dest. source and dest @@ -21,3 +25,60 @@ def copyfile(source, dest, buffer_size=1024 * 1024): source.close() dest.close() + + +def _lazy_load(fn): + _cached = [] + + def _decorated(): + if not _cached: + _cached.append(fn()) + return _cached[0] + return _decorated + + +@_lazy_load +def load_backend(): + from converter.conf.settings import GRAPHICS_BACKEND as backend_name + + try: + module = import_module('.base', 'converter.backends.%s' % backend_name) + import warnings + warnings.warn( + "Short names for CONVERTER_BACKEND are deprecated; prepend with 'converter.backends.'", + PendingDeprecationWarning + ) + return module + except ImportError, e: + # Look for a fully qualified converter backend name + try: + return import_module('.base', backend_name) + except ImportError, e_user: + # The converter backend wasn't found. Display a helpful error message + # listing all possible (built-in) converter backends. + backend_dir = os.path.join(os.path.dirname(__file__), 'backends') + try: + available_backends = [f for f in os.listdir(backend_dir) + if os.path.isdir(os.path.join(backend_dir, f)) + and not f.startswith('.')] + except EnvironmentError: + available_backends = [] + available_backends.sort() + if backend_name not in available_backends: + error_msg = ("%r isn't an available converter backend. \n" + + "Try using converter.backends.XXX, where XXX is one of:\n %s\n" + + "Error was: %s") % \ + (backend_name, ", ".join(map(repr, available_backends)), e_user) + raise ImproperlyConfigured(error_msg) + else: + raise # If there's some other error, this must be an error in Mayan itself. + + +def cleanup(filename): + """ + Tries to remove the given filename. Ignores non-existent files + """ + try: + os.remove(filename) + except OSError: + pass diff --git a/apps/converter/views.py b/apps/converter/views.py index ad95783539..ef7173f908 100644 --- a/apps/converter/views.py +++ b/apps/converter/views.py @@ -1,38 +1,18 @@ from django.utils.translation import ugettext_lazy as _ from django.shortcuts import render_to_response from django.template import RequestContext -from django.utils.importlib import import_module + +from converter import backend from converter.conf.settings import GRAPHICS_BACKEND - -def _lazy_load(fn): - _cached = [] - - def _decorated(): - if not _cached: - _cached.append(fn()) - return _cached[0] - return _decorated - - -@_lazy_load -def _get_backend(): - return import_module(GRAPHICS_BACKEND) - -try: - backend = _get_backend() -except ImportError: - raise ImportError(u'Missing or incorrect converter backend: %s' % GRAPHICS_BACKEND) - - def formats_list(request): #check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) context = { 'title': _(u'suported file formats'), 'hide_object': True, - 'object_list': backend.get_format_list(), + 'object_list': sorted(backend.get_format_list()), 'extra_columns': [ { 'name': _(u'name'), diff --git a/apps/documents/__init__.py b/apps/documents/__init__.py index ace7578129..162abfd605 100644 --- a/apps/documents/__init__.py +++ b/apps/documents/__init__.py @@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy as _ from django.core.urlresolvers import reverse from django.conf import settings +from common.utils import validate_path from navigation.api import register_links, register_top_menu, \ register_model_list_columns, register_multi_item_links, \ register_sidebar_template @@ -13,9 +14,6 @@ from metadata.api import get_metadata_string from documents.models import Document, DocumentPage, \ DocumentPageTransformation, DocumentType, DocumentTypeFilename -from documents.staging import StagingFile -from documents.conf.settings import USE_STAGING_DIRECTORY -from documents.conf.settings import PER_USER_STAGING_DIRECTORY from documents.literals import PERMISSION_DOCUMENT_CREATE, \ PERMISSION_DOCUMENT_PROPERTIES_EDIT, PERMISSION_DOCUMENT_VIEW, \ PERMISSION_DOCUMENT_DELETE, PERMISSION_DOCUMENT_DOWNLOAD, \ @@ -27,30 +25,9 @@ from documents.literals import HISTORY_DOCUMENT_CREATED, \ HISTORY_DOCUMENT_EDITED, HISTORY_DOCUMENT_DELETED from documents.conf.settings import ZOOM_MAX_LEVEL from documents.conf.settings import ZOOM_MIN_LEVEL +from documents.conf.settings import CACHE_PATH from documents.widgets import document_thumbnail -# Permission setup -set_namespace_title('documents', _(u'Documents')) -register_permission(PERMISSION_DOCUMENT_CREATE) -register_permission(PERMISSION_DOCUMENT_PROPERTIES_EDIT) -register_permission(PERMISSION_DOCUMENT_EDIT) -register_permission(PERMISSION_DOCUMENT_VIEW) -register_permission(PERMISSION_DOCUMENT_DELETE) -register_permission(PERMISSION_DOCUMENT_DOWNLOAD) -register_permission(PERMISSION_DOCUMENT_TRANSFORM) -register_permission(PERMISSION_DOCUMENT_TOOLS) - -# Document type permissions -register_permission(PERMISSION_DOCUMENT_TYPE_EDIT) -register_permission(PERMISSION_DOCUMENT_TYPE_DELETE) -register_permission(PERMISSION_DOCUMENT_TYPE_CREATE) - -# History setup -register_history_type(HISTORY_DOCUMENT_CREATED) -register_history_type(HISTORY_DOCUMENT_EDITED) -register_history_type(HISTORY_DOCUMENT_DELETED) - - # Document page links expressions def is_first_page(context): return context['object'].page_number <= 1 @@ -67,6 +44,28 @@ def is_min_zoom(context): def is_max_zoom(context): return context['zoom'] >= ZOOM_MAX_LEVEL +# Permission setup +set_namespace_title('documents', _(u'Documents')) +register_permission(PERMISSION_DOCUMENT_CREATE) +register_permission(PERMISSION_DOCUMENT_PROPERTIES_EDIT) +register_permission(PERMISSION_DOCUMENT_EDIT) +register_permission(PERMISSION_DOCUMENT_VIEW) +register_permission(PERMISSION_DOCUMENT_DELETE) +register_permission(PERMISSION_DOCUMENT_DOWNLOAD) +register_permission(PERMISSION_DOCUMENT_TRANSFORM) +register_permission(PERMISSION_DOCUMENT_TOOLS) + +# Document type permissions +set_namespace_title('documents_setup', _(u'Documents setup')) +register_permission(PERMISSION_DOCUMENT_TYPE_EDIT) +register_permission(PERMISSION_DOCUMENT_TYPE_DELETE) +register_permission(PERMISSION_DOCUMENT_TYPE_CREATE) + +# History setup +register_history_type(HISTORY_DOCUMENT_CREATED) +register_history_type(HISTORY_DOCUMENT_EDITED) +register_history_type(HISTORY_DOCUMENT_DELETED) + document_list = {'text': _(u'all documents'), 'view': 'document_list', 'famfam': 'page', 'permissions': [PERMISSION_DOCUMENT_VIEW]} document_list_recent = {'text': _(u'recent documents'), 'view': 'document_list_recent', 'famfam': 'page', 'permissions': [PERMISSION_DOCUMENT_VIEW]} document_create_multiple = {'text': _(u'upload new documents'), 'view': 'document_create_multiple', 'famfam': 'page_add', 'permissions': [PERMISSION_DOCUMENT_CREATE]} @@ -107,13 +106,6 @@ document_page_rotate_left = {'text': _(u'rotate left'), 'class': 'no-parent-hist document_missing_list = {'text': _(u'Find missing document files'), 'view': 'document_missing_list', 'famfam': 'folder_page', 'permissions': [PERMISSION_DOCUMENT_VIEW]} -upload_document_from_local = {'text': _(u'local'), 'view': 'upload_document_from_local', 'famfam': 'drive_disk', 'keep_query': True} -upload_document_from_staging = {'text': _(u'staging'), 'view': 'upload_document_from_staging', 'famfam': 'drive_network', 'keep_query': True, 'condition': lambda x: USE_STAGING_DIRECTORY} -upload_document_from_user_staging = {'text': _(u'user staging'), 'view': 'upload_document_from_user_staging', 'famfam': 'drive_user', 'keep_query': True, 'condition': lambda x: PER_USER_STAGING_DIRECTORY} - -staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source', 'object.id'], 'famfam': 'drive_magnify'} -staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source', 'object.id'], 'famfam': 'drive_delete'} - # Document type related links document_type_list = {'text': _(u'document type list'), 'view': 'document_type_list', 'famfam': 'layout', 'permissions': [PERMISSION_DOCUMENT_VIEW]} document_type_document_list = {'text': _(u'documents of this type'), 'view': 'document_type_document_list', 'args': 'object.id', 'famfam': 'page_go', 'permissions': [PERMISSION_DOCUMENT_VIEW]} @@ -139,9 +131,12 @@ register_links(['document_type_filename_edit', 'document_type_filename_delete'], # Register document links register_links(Document, [document_edit, document_print, document_delete, document_download, document_find_duplicates, document_clear_transformations, document_create_siblings]) -register_multi_item_links(['folder_view', 'index_instance_list', 'document_type_document_list', 'search', 'results', 'document_group_view', 'document_list', 'document_list_recent'], [document_multiple_clear_transformations, document_multiple_delete]) +register_multi_item_links(['document_find_duplicates', 'folder_view', 'index_instance_list', 'document_type_document_list', 'search', 'results', 'document_group_view', 'document_list', 'document_list_recent'], [document_multiple_clear_transformations, document_multiple_delete]) -register_links(['document_list_recent', 'document_list', 'document_create', 'document_create_multiple', 'upload_document', 'upload_document_from_local', 'upload_document_from_staging', 'upload_document_from_user_staging', 'document_find_duplicates'], [document_list_recent, document_list, document_create_multiple], menu_name='secondary_menu') +secondary_menu_links = [document_list_recent, document_list, document_create_multiple] + +register_links(['document_list_recent', 'document_list', 'document_create', 'document_create_multiple', 'upload_interactive', 'staging_file_delete'], secondary_menu_links, menu_name='secondary_menu') +#register_links(Document, secondary_menu_links, menu_name='sidebar') # Document page links register_links(DocumentPage, [ @@ -157,17 +152,12 @@ register_links(DocumentPage, [ register_links(['document_page_view'], [document_page_rotate_left, document_page_rotate_right, document_page_zoom_in, document_page_zoom_out], menu_name='form_header') -# Upload sources -register_links(['upload_document_from_local', 'upload_document_from_staging', 'upload_document_from_user_staging'], [upload_document_from_local, upload_document_from_staging, upload_document_from_user_staging], menu_name='form_header') - register_links(DocumentPageTransformation, [document_page_transformation_edit, document_page_transformation_delete]) register_links(DocumentPageTransformation, [document_page_transformation_page_edit, document_page_transformation_page_view], menu_name='sidebar') register_links('document_page_transformation_list', [document_page_transformation_create], menu_name='sidebar') register_links('document_page_transformation_create', [document_page_transformation_create], menu_name='sidebar') register_links(['document_page_transformation_edit', 'document_page_transformation_delete'], [document_page_transformation_page_transformation_list], menu_name='sidebar') -register_links(StagingFile, [staging_file_preview, staging_file_delete]) - register_diagnostic('documents', _(u'Documents'), document_missing_list) register_tool(document_find_all_duplicates, namespace='documents', title=_(u'documents')) @@ -209,3 +199,5 @@ register_sidebar_template(['document_type_list'], 'document_types_help.html') register_links(Document, [document_view_simple], menu_name='form_header', position=0) register_links(Document, [document_view_advanced], menu_name='form_header', position=1) register_links(Document, [document_history_view], menu_name='form_header') + +validate_path(CACHE_PATH) diff --git a/apps/documents/conf/settings.py b/apps/documents/conf/settings.py index 49af295531..5da5542b94 100644 --- a/apps/documents/conf/settings.py +++ b/apps/documents/conf/settings.py @@ -2,8 +2,10 @@ import hashlib import uuid +import os from django.utils.translation import ugettext_lazy as _ +from django.conf import settings from storage.backends.filebasedstorage import FileBasedStorage from smart_settings.api import register_settings @@ -18,30 +20,15 @@ def default_uuid(): """unicode(uuid.uuid4())""" return unicode(uuid.uuid4()) -available_transformations = { - 'rotate': {'label': _(u'Rotate [degrees]'), 'arguments': [{'name': 'degrees'}]} -} - register_settings( namespace=u'documents', module=u'documents.conf.settings', settings=[ - # Upload - {'name': u'USE_STAGING_DIRECTORY', 'global_name': u'DOCUMENTS_USE_STAGING_DIRECTORY', 'default': False}, - {'name': u'STAGING_DIRECTORY', 'global_name': u'DOCUMENTS_STAGING_DIRECTORY', 'default': u'/tmp/mayan/staging', 'exists': True}, - {'name': u'PER_USER_STAGING_DIRECTORY', 'global_name': u'DOCUMENTS_PER_USER_STAGING_DIRECTORY', 'default': False}, - {'name': u'USER_STAGING_DIRECTORY_ROOT', 'global_name': u'DOCUMENTS_USER_STAGING_DIRECTORY_ROOT', 'default': u'/tmp/mayan/staging/users', 'exists': True}, - {'name': u'USER_STAGING_DIRECTORY_EXPRESSION', 'global_name': u'DOCUMENTS_USER_STAGING_DIRECTORY_EXPRESSION', 'default': u'user.username'}, - {'name': u'DELETE_STAGING_FILE_AFTER_UPLOAD', 'global_name': u'DOCUMENTS_DELETE_STAGING_FILE_AFTER_UPLOAD', 'default': False}, - {'name': u'STAGING_FILES_PREVIEW_SIZE', 'global_name': u'DOCUMENTS_STAGING_FILES_PREVIEW_SIZE', 'default': u'640x480'}, # Saving {'name': u'CHECKSUM_FUNCTION', 'global_name': u'DOCUMENTS_CHECKSUM_FUNCTION', 'default': default_checksum}, {'name': u'UUID_FUNCTION', 'global_name': u'DOCUMENTS_UUID_FUNCTION', 'default': default_uuid}, # Storage {'name': u'STORAGE_BACKEND', 'global_name': u'DOCUMENTS_STORAGE_BACKEND', 'default': FileBasedStorage}, - # Transformations - {'name': u'AVAILABLE_TRANSFORMATIONS', 'global_name': u'DOCUMENTS_AVAILABLE_TRANSFORMATIONS', 'default': available_transformations}, - {'name': u'DEFAULT_TRANSFORMATIONS', 'global_name': u'DOCUMENTS_DEFAULT_TRANSFORMATIONS', 'default': []}, # Usage {'name': u'PREVIEW_SIZE', 'global_name': u'DOCUMENTS_PREVIEW_SIZE', 'default': u'640x480'}, {'name': u'PRINT_SIZE', 'global_name': u'DOCUMENTS_PRINT_SIZE', 'default': u'1400'}, @@ -53,5 +40,7 @@ register_settings( {'name': u'ZOOM_MAX_LEVEL', 'global_name': u'DOCUMENTS_ZOOM_MAX_LEVEL', 'default': 200, 'description': _(u'Maximum amount in percent (%) to allow user to zoom in a document page interactively.')}, {'name': u'ZOOM_MIN_LEVEL', 'global_name': u'DOCUMENTS_ZOOM_MIN_LEVEL', 'default': 50, 'description': _(u'Minimum amount in percent (%) to allow user to zoom out a document page interactively.')}, {'name': u'ROTATION_STEP', 'global_name': u'DOCUMENTS_ROTATION_STEP', 'default': 90, 'description': _(u'Amount in degrees to rotate a document page per user interaction.')}, + # + {'name': u'CACHE_PATH', 'global_name': u'DOCUMENTS_CACHE_PATH', 'default': os.path.join(settings.PROJECT_ROOT, 'image_cache'), 'exists': True}, ] ) diff --git a/apps/documents/forms.py b/apps/documents/forms.py index 9553b2ce00..650c677148 100644 --- a/apps/documents/forms.py +++ b/apps/documents/forms.py @@ -186,21 +186,11 @@ class DocumentForm(forms.ModelForm): queryset=filenames_qs, required=False, label=_(u'Quick document rename')) - - # Put the expand field last in the field order list - expand_field_index = self.fields.keyOrder.index('expand') - expand_field = self.fields.keyOrder.pop(expand_field_index) - self.fields.keyOrder.append(expand_field) new_filename = forms.CharField( label=_('New document filename'), required=False ) - - expand = forms.BooleanField( - label=_(u'Expand compressed files'), required=False, - help_text=ugettext(u'Upload a compressed file\'s contained files as individual documents') - ) - + class DocumentForm_edit(DocumentForm): """ @@ -208,12 +198,7 @@ class DocumentForm_edit(DocumentForm): """ class Meta: model = Document - exclude = ('file', 'document_type', 'tags', 'expand') - - - def __init__(self, *args, **kwargs): - super(DocumentForm_edit, self).__init__(*args, **kwargs) - self.fields.pop('expand') + exclude = ('file', 'document_type', 'tags') class DocumentPropertiesForm(DetailForm): @@ -266,32 +251,6 @@ class PrintForm(forms.Form): page_range = forms.CharField(label=_(u'Page range'), required=False) -class StagingDocumentForm(DocumentForm): - """ - Form that show all the files in the staging folder specified by the - StagingFile class passed as 'cls' argument - """ - def __init__(self, *args, **kwargs): - cls = kwargs.pop('cls') - super(StagingDocumentForm, self).__init__(*args, **kwargs) - try: - self.fields['staging_file_id'].choices = [ - (staging_file.id, staging_file) for staging_file in cls.get_all() - ] - except: - pass - - # Put staging_list field first in the field order list - staging_list_index = self.fields.keyOrder.index('staging_file_id') - staging_list = self.fields.keyOrder.pop(staging_list_index) - self.fields.keyOrder.insert(0, staging_list) - - staging_file_id = forms.ChoiceField(label=_(u'Staging file')) - - class Meta(DocumentForm.Meta): - exclude = ('description', 'file', 'document_type', 'tags') - - class DocumentTypeForm(forms.ModelForm): """ Model class form to create or edit a document type diff --git a/apps/documents/literals.py b/apps/documents/literals.py index aff31396f3..cbb3b919d9 100644 --- a/apps/documents/literals.py +++ b/apps/documents/literals.py @@ -14,13 +14,9 @@ PERMISSION_DOCUMENT_DOWNLOAD = {'namespace': 'documents', 'name': 'document_down PERMISSION_DOCUMENT_TRANSFORM = {'namespace': 'documents', 'name': 'document_transform', 'label': _(u'Transform documents')} PERMISSION_DOCUMENT_TOOLS = {'namespace': 'documents', 'name': 'document_tools', 'label': _(u'Execute document modifying tools')} -PERMISSION_DOCUMENT_TYPE_EDIT = {'namespace': 'documents', 'name': 'document_type_edit', 'label': _(u'Edit document types')} -PERMISSION_DOCUMENT_TYPE_DELETE = {'namespace': 'documents', 'name': 'document_type_delete', 'label': _(u'Delete document types')} -PERMISSION_DOCUMENT_TYPE_CREATE = {'namespace': 'documents', 'name': 'document_type_create', 'label': _(u'Create document types')} - -UPLOAD_SOURCE_LOCAL = u'local' -UPLOAD_SOURCE_STAGING = u'staging' -UPLOAD_SOURCE_USER_STAGING = u'user_staging' +PERMISSION_DOCUMENT_TYPE_EDIT = {'namespace': 'documents_setup', 'name': 'document_type_edit', 'label': _(u'Edit document types')} +PERMISSION_DOCUMENT_TYPE_DELETE = {'namespace': 'documents_setup', 'name': 'document_type_delete', 'label': _(u'Delete document types')} +PERMISSION_DOCUMENT_TYPE_CREATE = {'namespace': 'documents_setup', 'name': 'document_type_create', 'label': _(u'Create document types')} HISTORY_DOCUMENT_CREATED = { 'namespace': 'documents', 'name': 'document_created', diff --git a/apps/documents/managers.py b/apps/documents/managers.py index 3b007a936e..ef87c929fe 100644 --- a/apps/documents/managers.py +++ b/apps/documents/managers.py @@ -13,3 +13,24 @@ class RecentDocumentManager(models.Manager): to_delete = self.model.objects.filter(user=user)[RECENT_COUNT:] for recent_to_delete in to_delete: recent_to_delete.delete() + + +class DocumentPageTransformationManager(models.Manager): + def get_for_document_page(self, document_page): + return self.model.objects.filter(document_page=document_page) + + def get_for_document_page_as_list(self, document_page): + warnings = [] + transformations = [] + for transformation in self.get_for_document_page(document_page).values('transformation', 'arguments'): + try: + transformations.append( + { + 'transformation': transformation['transformation'], + 'arguments': eval(transformation['arguments'], {}) + } + ) + except Exception, e: + warnings.append(e) + + return transformations, warnings diff --git a/apps/documents/models.py b/apps/documents/models.py index c1a8771e38..6d8ed27e3b 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -1,26 +1,39 @@ import os import tempfile +import hashlib from django.db import models from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User from django.contrib.contenttypes import generic from django.contrib.comments.models import Comment +from django.conf import settings from taggit.managers import TaggableManager from dynamic_search.api import register from converter.api import get_page_count -from converter import TRANFORMATION_CHOICES +from converter.api import get_available_transformations_choices +from converter.api import create_image_cache_filename, convert +from converter.exceptions import UnknownFormat, UnkownConvertError from documents.utils import get_document_mimetype from documents.conf.settings import CHECKSUM_FUNCTION from documents.conf.settings import UUID_FUNCTION from documents.conf.settings import STORAGE_BACKEND -from documents.conf.settings import AVAILABLE_TRANSFORMATIONS -from documents.conf.settings import DEFAULT_TRANSFORMATIONS -from documents.managers import RecentDocumentManager +from documents.conf.settings import PREVIEW_SIZE +from documents.conf.settings import THUMBNAIL_SIZE +from documents.conf.settings import CACHE_PATH -available_transformations = ([(name, data['label']) for name, data in AVAILABLE_TRANSFORMATIONS.items()]) +from documents.managers import RecentDocumentManager, \ + DocumentPageTransformationManager +from documents.utils import document_save_to_temp_dir +from documents.literals import PICTURE_ERROR_SMALL, PICTURE_ERROR_MEDIUM, \ + PICTURE_UNKNOWN_SMALL, PICTURE_UNKNOWN_MEDIUM +from converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, \ + DEFAULT_FILE_FORMAT, DEFAULT_PAGE_NUMBER + +# document image cache name hash function +HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() def get_filename_from_uuid(instance, filename): @@ -92,7 +105,7 @@ class Document(models.Model): mimetype, page count and transformation when originally created """ new_document = not self.pk - + transformations = kwargs.pop('transformations', None) super(Document, self).save(*args, **kwargs) if new_document: @@ -101,7 +114,8 @@ class Document(models.Model): self.update_mimetype(save=False) self.save() self.update_page_count(save=False) - self.apply_default_transformations() + if transformations: + self.apply_default_transformations(transformations) @models.permalink def get_absolute_url(self): @@ -195,21 +209,43 @@ class Document(models.Model): exists in storage """ return self.file.storage.exists(self.file.path) - - def apply_default_transformations(self): + + def apply_default_transformations(self, transformations): #Only apply default transformations on new documents - if DEFAULT_TRANSFORMATIONS and reduce(lambda x, y: x + y, [page.documentpagetransformation_set.count() for page in self.documentpage_set.all()]) == 0: - for transformation in DEFAULT_TRANSFORMATIONS: - if 'name' in transformation: - for document_page in self.documentpage_set.all(): - page_transformation = DocumentPageTransformation( - document_page=document_page, - order=0, - transformation=transformation['name']) - if 'arguments' in transformation: - page_transformation.arguments = transformation['arguments'] + if reduce(lambda x, y: x + y, [page.documentpagetransformation_set.count() for page in self.documentpage_set.all()]) == 0: + for transformation in transformations: + for document_page in self.documentpage_set.all(): + page_transformation = DocumentPageTransformation( + document_page=document_page, + order=0, + transformation=transformation.get('transformation'), + arguments=transformation.get('arguments') + ) - page_transformation.save() + page_transformation.save() + + def get_image_cache_name(self, page): + document_page = self.documentpage_set.get(page_number=page) + transformations, warnings = document_page.get_transformation_list() + hash_value = HASH_FUNCTION(u''.join([self.checksum, unicode(page), unicode(transformations)])) + cache_file_path = os.path.join(CACHE_PATH, hash_value) + if os.path.exists(cache_file_path): + return cache_file_path + else: + document_file = document_save_to_temp_dir(self, self.checksum) + return convert(document_file, output_filepath=cache_file_path, page=page, transformations=transformations) + + def get_image(self, size=PREVIEW_SIZE, page=DEFAULT_PAGE_NUMBER, zoom=DEFAULT_ZOOM_LEVEL, rotation=DEFAULT_ROTATION): + try: + image_cache_name = self.get_image_cache_name(page=page) + output_file = convert(image_cache_name, cleanup_files=False, size=size, zoom=zoom, rotation=rotation) + except UnknownFormat: + output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_UNKNOWN_SMALL) + except UnkownConvertError: + output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_SMALL) + except Exception, e: + output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_SMALL) + return output_file class DocumentTypeFilename(models.Model): @@ -251,26 +287,13 @@ class DocumentPage(models.Model): verbose_name = _(u'document page') verbose_name_plural = _(u'document pages') + def get_transformation_list(self): + return DocumentPageTransformation.objects.get_for_document_page_as_list(self) + @models.permalink def get_absolute_url(self): return ('document_page_view', [self.pk]) - def get_transformation_string(self): - transformation_list = [] - warnings = [] - for page_transformation in self.documentpagetransformation_set.all(): - try: - if page_transformation.transformation in TRANFORMATION_CHOICES: - transformation_list.append( - TRANFORMATION_CHOICES[page_transformation.transformation] % eval( - page_transformation.arguments - ) - ) - except Exception, e: - warnings.append(e) - - return u' '.join(transformation_list), warnings - class DocumentPageTransformation(models.Model): """ @@ -279,9 +302,11 @@ class DocumentPageTransformation(models.Model): """ document_page = models.ForeignKey(DocumentPage, verbose_name=_(u'document page')) order = models.PositiveIntegerField(default=0, blank=True, null=True, verbose_name=_(u'order'), db_index=True) - transformation = models.CharField(choices=available_transformations, max_length=128, verbose_name=_(u'transformation')) + transformation = models.CharField(choices=get_available_transformations_choices(), max_length=128, verbose_name=_(u'transformation')) arguments = models.TextField(blank=True, null=True, verbose_name=_(u'arguments'), help_text=_(u'Use dictionaries to indentify arguments, example: {\'degrees\':90}')) + objects = DocumentPageTransformationManager() + def __unicode__(self): return u'"%s" for %s' % (self.get_transformation_display(), unicode(self.document_page)) diff --git a/apps/documents/templatetags/printing_tags.py b/apps/documents/templatetags/printing_tags.py index 33f560bdf1..d556f6e10d 100644 --- a/apps/documents/templatetags/printing_tags.py +++ b/apps/documents/templatetags/printing_tags.py @@ -1,6 +1,6 @@ from django.template import Library, Node, Variable -from converter.api import get_document_dimensions, QUALITY_PRINT +from converter.api import get_document_dimensions from documents.views import calculate_converter_arguments from documents.conf.settings import PRINT_SIZE @@ -14,8 +14,7 @@ class GetImageSizeNode(Node): def render(self, context): document = Variable(self.document).resolve(context) - arguments, warnings = calculate_converter_arguments(document, size=PRINT_SIZE, quality=QUALITY_PRINT) - width, height = get_document_dimensions(document, **arguments) + width, height = get_document_dimensions(document) context[u'document_width'], context['document_height'] = width, height context[u'document_aspect'] = float(width) / float(height) return u'' diff --git a/apps/documents/urls.py b/apps/documents/urls.py index 1add9cd55b..19020a3448 100644 --- a/apps/documents/urls.py +++ b/apps/documents/urls.py @@ -1,24 +1,16 @@ from django.conf.urls.defaults import patterns, url -from converter.api import QUALITY_HIGH, QUALITY_PRINT - from documents.conf.settings import PREVIEW_SIZE from documents.conf.settings import PRINT_SIZE from documents.conf.settings import THUMBNAIL_SIZE from documents.conf.settings import DISPLAY_SIZE from documents.conf.settings import MULTIPAGE_PREVIEW_SIZE -from documents.literals import UPLOAD_SOURCE_LOCAL, \ - UPLOAD_SOURCE_STAGING, UPLOAD_SOURCE_USER_STAGING urlpatterns = patterns('documents.views', url(r'^list/$', 'document_list', (), 'document_list'), url(r'^list/recent/$', 'document_list_recent', (), 'document_list_recent'), url(r'^create/from/local/multiple/$', 'document_create', (), 'document_create_multiple'), - url(r'^upload/local/$', 'upload_document_with_type', {'source': UPLOAD_SOURCE_LOCAL}, 'upload_document_from_local'), - url(r'^upload/staging/$', 'upload_document_with_type', {'source': UPLOAD_SOURCE_STAGING}, 'upload_document_from_staging'), - url(r'^upload/staging/user/$', 'upload_document_with_type', {'source': UPLOAD_SOURCE_USER_STAGING}, 'upload_document_from_user_staging'), - url(r'^(?P\d+)/view/$', 'document_view', (), 'document_view_simple'), url(r'^(?P\d+)/view/advanced/$', 'document_view', {'advanced': True}, 'document_view_advanced'), url(r'^(?P\d+)/delete/$', 'document_delete', (), 'document_delete'), @@ -30,8 +22,8 @@ urlpatterns = patterns('documents.views', url(r'^(?P\d+)/display/preview/$', 'get_document_image', {'size': PREVIEW_SIZE}, 'document_preview'), url(r'^(?P\d+)/display/preview/multipage/$', 'get_document_image', {'size': MULTIPAGE_PREVIEW_SIZE}, 'document_preview_multipage'), url(r'^(?P\d+)/display/thumbnail/$', 'get_document_image', {'size': THUMBNAIL_SIZE}, 'document_thumbnail'), - url(r'^(?P\d+)/display/$', 'get_document_image', {'size': DISPLAY_SIZE, 'quality': QUALITY_HIGH}, 'document_display'), - url(r'^(?P\d+)/display/print/$', 'get_document_image', {'size': PRINT_SIZE, 'quality': QUALITY_PRINT}, 'document_display_print'), + url(r'^(?P\d+)/display/$', 'get_document_image', {'size': DISPLAY_SIZE}, 'document_display'), + url(r'^(?P\d+)/display/print/$', 'get_document_image', {'size': PRINT_SIZE}, 'document_display_print'), url(r'^(?P\d+)/download/$', 'document_download', (), 'document_download'), url(r'^(?P\d+)/create/siblings/$', 'document_create_siblings', (), 'document_create_siblings'), @@ -41,9 +33,6 @@ urlpatterns = patterns('documents.views', url(r'^multiple/clear_transformations/$', 'document_multiple_clear_transformations', (), 'document_multiple_clear_transformations'), url(r'^duplicates/list/$', 'document_find_all_duplicates', (), 'document_find_all_duplicates'), - url(r'^staging_file/type/(?P\w+)/(?P\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), - url(r'^staging_file/type/(?P\w+)/(?P\w+)/delete/$', 'staging_file_delete', (), 'staging_file_delete'), - url(r'^page/(?P\d+)/$', 'document_page_view', (), 'document_page_view'), url(r'^page/(?P\d+)/text/$', 'document_page_text', (), 'document_page_text'), url(r'^page/(?P\d+)/edit/$', 'document_page_edit', (), 'document_page_edit'), diff --git a/apps/documents/utils.py b/apps/documents/utils.py index 9123a5b380..1a77975100 100644 --- a/apps/documents/utils.py +++ b/apps/documents/utils.py @@ -1,6 +1,6 @@ import os -from common import TEMPORARY_DIRECTORY +from common.conf.settings import TEMPORARY_DIRECTORY try: from python_magic import magic diff --git a/apps/documents/views.py b/apps/documents/views.py index d29e9eda18..f86a73f3d8 100644 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -1,5 +1,4 @@ import os -import zipfile import urlparse import copy @@ -13,7 +12,6 @@ from django.core.urlresolvers import reverse from django.views.generic.create_update import delete_object, update_object from django.conf import settings from django.utils.http import urlencode -from django.core.files.uploadedfile import SimpleUploadedFile import sendfile from common.utils import pretty_size, parse_range, urlquote, \ @@ -22,10 +20,8 @@ from common.widgets import two_state_template from common.literals import PAGE_SIZE_DIMENSIONS, \ PAGE_ORIENTATION_PORTRAIT, PAGE_ORIENTATION_LANDSCAPE from common.conf.settings import DEFAULT_PAPER_SIZE -from converter.api import convert_document, QUALITY_DEFAULT -from converter.exceptions import UnkownConvertError, UnknownFormat -from converter.api import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, \ - DEFAULT_FILE_FORMAT, QUALITY_PRINT +from converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, \ + DEFAULT_FILE_FORMAT, DEFAULT_PAGE_NUMBER from filetransfers.api import serve_file from grouping.utils import get_document_group_subtemplate from metadata.api import save_metadata_list, \ @@ -36,10 +32,6 @@ from permissions.api import check_permissions from document_indexing.api import update_indexes, delete_indexes from history.api import create_history -from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD -from documents.conf.settings import USE_STAGING_DIRECTORY -from documents.conf.settings import PER_USER_STAGING_DIRECTORY - from documents.conf.settings import PREVIEW_SIZE from documents.conf.settings import THUMBNAIL_SIZE from documents.conf.settings import STORAGE_BACKEND @@ -61,7 +53,7 @@ from documents.literals import HISTORY_DOCUMENT_CREATED, \ from documents.forms import DocumentTypeSelectForm, \ DocumentForm, DocumentForm_edit, DocumentPropertiesForm, \ - StagingDocumentForm, DocumentPreviewForm, \ + DocumentPreviewForm, \ DocumentPageForm, DocumentPageTransformationForm, \ DocumentContentForm, DocumentPageForm_edit, \ DocumentPageForm_text, PrintForm, DocumentTypeForm, \ @@ -69,11 +61,8 @@ from documents.forms import DocumentTypeSelectForm, \ from documents.wizards import DocumentCreateWizard from documents.models import Document, DocumentType, DocumentPage, \ DocumentPageTransformation, RecentDocument, DocumentTypeFilename -from documents.staging import create_staging_file_class from documents.literals import PICTURE_ERROR_SMALL, PICTURE_ERROR_MEDIUM, \ PICTURE_UNKNOWN_SMALL, PICTURE_UNKNOWN_MEDIUM -from documents.literals import UPLOAD_SOURCE_LOCAL, \ - UPLOAD_SOURCE_STAGING, UPLOAD_SOURCE_USER_STAGING # Document type permissions from documents.literals import PERMISSION_DOCUMENT_TYPE_EDIT, \ @@ -116,171 +105,10 @@ def document_create_siblings(request, document_id): if document.document_type_id: query_dict['document_type_id'] = document.document_type_id - url = reverse('upload_document_from_local') + url = reverse('upload_interactive') return HttpResponseRedirect('%s?%s' % (url, urlencode(query_dict))) -def _handle_save_document(request, document, form=None): - RecentDocument.objects.add_document_for_user(request.user, document) - - if form: - if form.cleaned_data['new_filename']: - document.file_filename = form.cleaned_data['new_filename'] - document.save() - - if form and 'document_type_available_filenames' in form.cleaned_data: - if form.cleaned_data['document_type_available_filenames']: - document.file_filename = form.cleaned_data['document_type_available_filenames'].filename - document.save() - - save_metadata_list(decode_metadata_from_url(request.GET), document, create=True) - - warnings = update_indexes(document) - if request.user.is_staff or request.user.is_superuser: - for warning in warnings: - messages.warning(request, warning) - - create_history(HISTORY_DOCUMENT_CREATED, document, {'user': request.user}) - - -def _handle_zip_file(request, uploaded_file, document_type=None): - filename = getattr(uploaded_file, 'filename', getattr(uploaded_file, 'name', '')) - if filename.lower().endswith('zip'): - zfobj = zipfile.ZipFile(uploaded_file) - for filename in zfobj.namelist(): - if not filename.endswith('/'): - zip_document = Document(file=SimpleUploadedFile( - name=filename, content=zfobj.read(filename))) - if document_type: - zip_document.document_type = document_type - zip_document.save() - _handle_save_document(request, zip_document) - messages.success(request, _(u'Extracted file: %s, uploaded successfully.') % filename) - #Signal that uploaded file was a zip file - return True - else: - #Otherwise tell parent to handle file - return False - - -def upload_document_with_type(request, source): - check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE]) - - document_type_id = request.GET.get('document_type_id', None) - if document_type_id: - document_type = get_object_or_404(DocumentType, pk=document_type_id[0]) - else: - document_type = None - - if request.method == 'POST': - if source == UPLOAD_SOURCE_LOCAL: - form = DocumentForm(request.POST, request.FILES, document_type=document_type) - if form.is_valid(): - try: - expand = form.cleaned_data['expand'] - if (not expand) or (expand and not _handle_zip_file(request, request.FILES['file'], document_type)): - instance = form.save() - instance.save() - if document_type: - instance.document_type = document_type - _handle_save_document(request, instance, form) - messages.success(request, _(u'Document uploaded successfully.')) - except Exception, e: - messages.error(request, e) - - return HttpResponseRedirect(request.get_full_path()) - elif (USE_STAGING_DIRECTORY and source == UPLOAD_SOURCE_STAGING) or (PER_USER_STAGING_DIRECTORY and source == UPLOAD_SOURCE_USER_STAGING): - StagingFile = create_staging_file_class(request, source) - form = StagingDocumentForm(request.POST, - request.FILES, cls=StagingFile, - document_type=document_type) - if form.is_valid(): - try: - staging_file = StagingFile.get(form.cleaned_data['staging_file_id']) - expand = form.cleaned_data['expand'] - if (not expand) or (expand and not _handle_zip_file(request, staging_file.upload(), document_type)): - document = Document(file=staging_file.upload()) - if document_type: - document.document_type = document_type - document.save() - _handle_save_document(request, document, form) - messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename) - - if DELETE_STAGING_FILE_AFTER_UPLOAD: - staging_file.delete() - messages.success(request, _(u'Staging file: %s, deleted successfully.') % staging_file.filename) - except Exception, e: - messages.error(request, e) - - return HttpResponseRedirect(request.META['HTTP_REFERER']) - else: - if source == UPLOAD_SOURCE_LOCAL: - form = DocumentForm(document_type=document_type) - elif (USE_STAGING_DIRECTORY and source == UPLOAD_SOURCE_STAGING) or (PER_USER_STAGING_DIRECTORY and source == UPLOAD_SOURCE_USER_STAGING): - StagingFile = create_staging_file_class(request, source) - form = StagingDocumentForm(cls=StagingFile, - document_type=document_type) - - subtemplates_list = [] - - if source == UPLOAD_SOURCE_LOCAL: - subtemplates_list.append({ - 'name': 'generic_form_subtemplate.html', - 'context': { - 'form': form, - 'title': _(u'upload a local document'), - }, - }) - - elif (USE_STAGING_DIRECTORY and source == UPLOAD_SOURCE_STAGING) or (PER_USER_STAGING_DIRECTORY and source == UPLOAD_SOURCE_USER_STAGING): - if source == UPLOAD_SOURCE_STAGING: - form_title = _(u'upload a document from staging') - list_title = _(u'files in staging') - else: - form_title = _(u'upload a document from user staging') - list_title = _(u'files in user staging') - try: - staging_filelist = StagingFile.get_all() - except Exception, e: - messages.error(request, e) - staging_filelist = [] - finally: - subtemplates_list = [ - { - 'name': 'generic_form_subtemplate.html', - 'context': { - 'form': form, - 'title': form_title, - } - }, - { - 'name': 'generic_list_subtemplate.html', - 'context': { - 'title': list_title, - 'object_list': staging_filelist, - 'hide_link': True, - } - }, - ] - - context = { - 'source': source, - 'document_type_id': document_type_id, - 'subtemplates_list': subtemplates_list, - 'sidebar_subtemplates_list': [ - { - 'name': 'generic_subtemplate.html', - 'context': { - 'title': _(u'Current metadata'), - 'paragraphs': metadata_repr_as_list(decode_metadata_from_url(request.GET)), - 'side_bar': True, - } - }] - } - return render_to_response('generic_form.html', context, - context_instance=RequestContext(request)) - - def document_view(request, document_id, advanced=False): check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) #document = get_object_or_404(Document.objects.select_related(), pk=document_id) @@ -456,38 +284,14 @@ def document_edit(request, document_id): }, context_instance=RequestContext(request)) -def calculate_converter_arguments(document, *args, **kwargs): - size = kwargs.pop('size', PREVIEW_SIZE) - quality = kwargs.pop('quality', QUALITY_DEFAULT) - page = kwargs.pop('page', 1) - file_format = kwargs.pop('file_format', DEFAULT_FILE_FORMAT) - zoom = kwargs.pop('zoom', DEFAULT_ZOOM_LEVEL) - rotation = kwargs.pop('rotation', DEFAULT_ROTATION) - - document_page = DocumentPage.objects.get(document=document, page_number=page) - transformation_string, warnings = document_page.get_transformation_string() - - arguments = { - 'size': size, - 'file_format': file_format, - 'quality': quality, - 'extra_options': transformation_string, - 'page': page - 1, - 'zoom': zoom, - 'rotation': rotation - } - - return arguments, warnings - - -def get_document_image(request, document_id, size=PREVIEW_SIZE, quality=QUALITY_DEFAULT): +def get_document_image(request, document_id, size=PREVIEW_SIZE): check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) document = get_object_or_404(Document, pk=document_id) - page = int(request.GET.get('page', 1)) + page = int(request.GET.get('page', DEFAULT_PAGE_NUMBER)) - zoom = int(request.GET.get('zoom', 100)) + zoom = int(request.GET.get('zoom', DEFAULT_ZOOM_LEVEL)) if zoom < ZOOM_MIN_LEVEL: zoom = ZOOM_MIN_LEVEL @@ -495,37 +299,9 @@ def get_document_image(request, document_id, size=PREVIEW_SIZE, quality=QUALITY_ if zoom > ZOOM_MAX_LEVEL: zoom = ZOOM_MAX_LEVEL - rotation = int(request.GET.get('rotation', 0)) % 360 + rotation = int(request.GET.get('rotation', DEFAULT_ROTATION)) % 360 - arguments, warnings = calculate_converter_arguments(document, size=size, file_format=DEFAULT_FILE_FORMAT, quality=quality, page=page, zoom=zoom, rotation=rotation) - - if warnings and (request.user.is_staff or request.user.is_superuser): - for warning in warnings: - messages.warning(request, _(u'Page transformation error: %s') % warning) - - try: - output_file = convert_document(document, **arguments) - except UnkownConvertError, e: - if request.user.is_staff or request.user.is_superuser: - messages.error(request, e) - if size == THUMBNAIL_SIZE: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_SMALL) - else: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_MEDIUM) - except UnknownFormat: - if size == THUMBNAIL_SIZE: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_UNKNOWN_SMALL) - else: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_UNKNOWN_MEDIUM) - except Exception, e: - if request.user.is_staff or request.user.is_superuser: - messages.error(request, e) - if size == THUMBNAIL_SIZE: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_SMALL) - else: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_MEDIUM) - finally: - return sendfile.sendfile(request, output_file) + return sendfile.sendfile(request, document.get_image(size=size, page=page, zoom=zoom, rotation=rotation)) def document_download(request, document_id): @@ -546,58 +322,6 @@ def document_download(request, document_id): return HttpResponseRedirect(request.META['HTTP_REFERER']) -def staging_file_preview(request, source, staging_file_id): - check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE]) - StagingFile = create_staging_file_class(request, source) - try: - output_file, errors = StagingFile.get(staging_file_id).preview() - if errors and (request.user.is_staff or request.user.is_superuser): - for error in errors: - messages.warning(request, _(u'Staging file transformation error: %(error)s') % { - 'error': error - }) - - except UnkownConvertError, e: - if request.user.is_staff or request.user.is_superuser: - messages.error(request, e) - - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_MEDIUM) - except UnknownFormat: - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_UNKNOWN_MEDIUM) - except Exception, e: - if request.user.is_staff or request.user.is_superuser: - messages.error(request, e) - output_file = os.path.join(settings.MEDIA_ROOT, u'images', PICTURE_ERROR_MEDIUM) - finally: - return sendfile.sendfile(request, output_file) - - -def staging_file_delete(request, source, staging_file_id): - check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE]) - StagingFile = create_staging_file_class(request, source) - - staging_file = StagingFile.get(staging_file_id) - next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None))) - previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None))) - - if request.method == 'POST': - try: - staging_file.delete() - messages.success(request, _(u'Staging file delete successfully.')) - except Exception, e: - messages.error(request, e) - return HttpResponseRedirect(next) - - return render_to_response('generic_confirm.html', { - 'source': source, - 'delete_view': True, - 'object': staging_file, - 'next': next, - 'previous': previous, - 'form_icon': u'drive_delete.png', - }, context_instance=RequestContext(request)) - - def document_page_transformation_list(request, document_page_id): check_permissions(request.user, [PERMISSION_DOCUMENT_TRANSFORM]) @@ -689,10 +413,14 @@ def document_find_duplicates(request, document_id): check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) document = get_object_or_404(Document, pk=document_id) - return _find_duplicate_list(request, [document], include_source=True, confirmation=False) + extra_context = { + 'title': _(u'duplicates of: %s') % document, + 'object': document, + } + return _find_duplicate_list(request, [document], include_source=True, confirmation=False, extra_context=extra_context) -def _find_duplicate_list(request, source_document_list=Document.objects.all(), include_source=False, confirmation=True): +def _find_duplicate_list(request, source_document_list=Document.objects.all(), include_source=False, confirmation=True, extra_context=None): previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None))) if confirmation and request.method != 'POST': @@ -712,10 +440,18 @@ def _find_duplicate_list(request, source_document_list=Document.objects.all(), i if include_source and results: duplicated.append(document.pk) - return render_to_response('generic_list.html', { + context = { 'object_list': Document.objects.filter(pk__in=duplicated), 'title': _(u'duplicated documents'), - }, context_instance=RequestContext(request)) + 'hide_links': True, + 'multi_select_as_buttons': True, + } + + if extra_context: + context.update(extra_context) + + return render_to_response('generic_list.html', context, + context_instance=RequestContext(request)) def document_find_all_duplicates(request): @@ -802,13 +538,13 @@ def document_page_view(request, document_page_id): document_page = get_object_or_404(DocumentPage, pk=document_page_id) - zoom = int(request.GET.get('zoom', 100)) - rotation = int(request.GET.get('rotation', 0)) + zoom = int(request.GET.get('zoom', DEFAULT_ZOOM_LEVEL)) + rotation = int(request.GET.get('rotation', DEFAULT_ROTATION)) document_page_form = DocumentPageForm(instance=document_page, zoom=zoom, rotation=rotation) base_title = _(u'details for: %s') % document_page - if zoom != 100: + if zoom != DEFAULT_ZOOM_LEVEL: zoom_text = u'(%d%%)' % zoom else: zoom_text = u'' @@ -1036,13 +772,14 @@ def document_print(request, document_id): def document_hard_copy(request, document_id): + #TODO: FIXME check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) document = get_object_or_404(Document, pk=document_id) RecentDocument.objects.add_document_for_user(request.user, document) - arguments, warnings = calculate_converter_arguments(document, size=PRINT_SIZE, file_format=DEFAULT_FILE_FORMAT, quality=QUALITY_PRINT) + arguments, warnings = calculate_converter_arguments(document, size=PRINT_SIZE, file_format=DEFAULT_FILE_FORMAT) # Pre-generate convert_document(document, **arguments) diff --git a/apps/documents/wizards.py b/apps/documents/wizards.py index c2b76b9028..a61ba4960a 100644 --- a/apps/documents/wizards.py +++ b/apps/documents/wizards.py @@ -30,7 +30,6 @@ class DocumentCreateWizard(BoundFormWizard): def __init__(self, *args, **kwargs): self.query_dict = {} - self.multiple = kwargs.pop('multiple', True) self.step_titles = kwargs.pop('step_titles', [ _(u'step 1 of 3: Document type'), _(u'step 2 of 3: Metadata selection'), @@ -75,13 +74,8 @@ class DocumentCreateWizard(BoundFormWizard): return 'generic_wizard.html' def done(self, request, form_list): - if self.multiple: - view = 'upload_document_from_local' - else: - view = 'upload_document' - if self.document_type: self.query_dict['document_type_id'] = self.document_type.pk - url = urlquote(reverse(view), self.query_dict) + url = urlquote(reverse('upload_interactive'), self.query_dict) return HttpResponseRedirect(url) diff --git a/apps/main/__init__.py b/apps/main/__init__.py index c25a86ec9e..7f9364129a 100644 --- a/apps/main/__init__.py +++ b/apps/main/__init__.py @@ -9,6 +9,7 @@ from converter import formats_list from documents import document_type_views from metadata import setup_metadata_type_list, metadata_type_setup_views from metadata import setup_metadata_set_list, metadata_set_setup_views +from sources import source_list, source_views from main.conf.settings import SIDE_BAR_SEARCH from main.conf.settings import DISABLE_HOME_VIEW @@ -45,18 +46,19 @@ if not SIDE_BAR_SEARCH: register_top_menu('tools', link=tools_menu, children_views=['statistics', 'history_list', 'formats_list']) #register_top_menu('setup_menu', link={'text': _(u'setup'), 'view': 'setting_list', 'famfam': 'cog'}, children=setup_views) -register_top_menu('setup_menu', link={'text': _(u'setup'), 'view': 'setting_list', 'famfam': 'cog'}, children_path_regex=[r'^settings/', r'^user_management/', r'^permissions', r'^documents/type', r'^metadata/setup']) +register_top_menu('setup_menu', link={'text': _(u'setup'), 'view': 'setting_list', 'famfam': 'cog'}, children_path_regex=[r'^settings/', r'^user_management/', r'^permissions', r'^documents/type', r'^metadata/setup', r'sources/setup']) register_top_menu('about', link={'text': _(u'about'), 'view': 'about', 'famfam': 'information'}) register_links(['tools_menu', 'statistics', 'history_list', 'history_view', 'formats_list'], [tools_menu, statistics, history_list, formats_list, sentry], menu_name='secondary_menu') -setup_links = [check_settings, role_list, user_list, group_list, document_types, setup_metadata_type_list, setup_metadata_set_list, admin_site] +setup_links = [check_settings, role_list, user_list, group_list, document_types, setup_metadata_type_list, setup_metadata_set_list, source_list, admin_site] register_links(['setting_list'], setup_links, menu_name='secondary_menu') register_links(permission_views, setup_links, menu_name='secondary_menu') register_links(user_management_views, setup_links, menu_name='secondary_menu') register_links(document_type_views, setup_links, menu_name='secondary_menu') register_links(metadata_type_setup_views, setup_links, menu_name='secondary_menu') register_links(metadata_set_setup_views, setup_links, menu_name='secondary_menu') +register_links(source_views, setup_links, menu_name='secondary_menu') def get_version(): diff --git a/apps/main/templates/base.html b/apps/main/templates/base.html index deb49781c8..5c2c4ca43a 100644 --- a/apps/main/templates/base.html +++ b/apps/main/templates/base.html @@ -5,6 +5,8 @@ {% load settings %} {% load search_tags %} {% load main_settings_tags %} +{% load variable_tags %} + {% block web_theme_head %} {% if new_window_url %}