From ef577e5c77fdb151ac198bf93b8ec59e6b311214 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 7 Jul 2014 02:41:02 -0400 Subject: [PATCH 01/29] Remove unused URL configs --- mayan/apps/documents/urls.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index c48976263e..06aea0cf7d 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -2,8 +2,8 @@ from __future__ import absolute_import from django.conf.urls import patterns, url -from .conf.settings import (PREVIEW_SIZE, PRINT_SIZE, THUMBNAIL_SIZE, - DISPLAY_SIZE, MULTIPAGE_PREVIEW_SIZE) +from .conf.settings import (PREVIEW_SIZE, PRINT_SIZE, DISPLAY_SIZE, + MULTIPAGE_PREVIEW_SIZE) urlpatterns = patterns('documents.views', url(r'^list/$', 'document_list', (), 'document_list'), @@ -19,14 +19,9 @@ urlpatterns = patterns('documents.views', url(r'^(?P\d+)/display/preview/$', 'get_document_image', {'size': PREVIEW_SIZE}, 'document_preview'), url(r'^(?P\d+)/display/preview/multipage/$', 'get_document_image', {'size': MULTIPAGE_PREVIEW_SIZE}, 'document_preview_multipage'), - url(r'^(?P\d+)/display/thumbnail/$', 'get_document_image', {'size': THUMBNAIL_SIZE}, 'document_thumbnail'), url(r'^(?P\d+)/display/$', 'get_document_image', {'size': DISPLAY_SIZE}, 'document_display'), url(r'^(?P\d+)/display/print/$', 'get_document_image', {'size': PRINT_SIZE}, 'document_display_print'), - url(r'^(?P\d+)/display/preview/base64/$', 'get_document_image', {'size': PREVIEW_SIZE, 'base64_version': True}, 'document_preview_base64'), - url(r'^(?P\d+)/display/preview/multipage/base64/$', 'get_document_image', {'size': MULTIPAGE_PREVIEW_SIZE, 'base64_version': True}, 'document_preview_multipage_base64'), - url(r'^(?P\d+)/display/thumbnail/base64/$', 'get_document_image', {'size': THUMBNAIL_SIZE, 'base64_version': True}, 'document_thumbnail_base64'), - url(r'^(?P\d+)/download/$', 'document_download', (), 'document_download'), url(r'^multiple/download/$', 'document_multiple_download', (), 'document_multiple_download'), url(r'^(?P\d+)/find_duplicates/$', 'document_find_duplicates', (), 'document_find_duplicates'), From a42df480e9abbd0b12b4ab0c3339039f20fcd658 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 7 Jul 2014 02:41:27 -0400 Subject: [PATCH 02/29] Remove remarked code --- .../apps/documents/templates/document_print.html | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/mayan/apps/documents/templates/document_print.html b/mayan/apps/documents/templates/document_print.html index 66841fd6e6..cdae522fcf 100644 --- a/mayan/apps/documents/templates/document_print.html +++ b/mayan/apps/documents/templates/document_print.html @@ -1,5 +1,5 @@ {% load project_tags %} -{#{% load printing_tags %}#} + @@ -13,7 +13,7 @@ padding: 0; margin: 0; background: #fff; - color: #000; + color: #000; } html, body { height: 100%; @@ -25,7 +25,7 @@ } * html .container { height: 100%; - } + } .centered { position: absolute; top: 0; @@ -33,7 +33,7 @@ bottom: 0; left: 0; margin: auto; - } + } {% endcomment %} .break { page-break-after: always; } img { border: 1px solid black; } @@ -45,15 +45,12 @@ margin-right: auto; } - - + + {% for page in pages %} - {#{% get_document_size object %}#}
- {# page_aspect %}width="97%"{% else %}height="97%"{% endif %} />#}
{% endfor %} - From be5525496eb69efd30f5223ddaf2ca7aec89d717 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 7 Jul 2014 02:47:03 -0400 Subject: [PATCH 03/29] Fix document version serialization --- mayan/apps/documents/api.py | 3 +-- mayan/apps/documents/resources.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/mayan/apps/documents/api.py b/mayan/apps/documents/api.py index 270156f66f..eccb7ff2f5 100644 --- a/mayan/apps/documents/api.py +++ b/mayan/apps/documents/api.py @@ -38,11 +38,10 @@ class APIDocumentView(generics.RetrieveAPIView): class APIDocumentVersionView(generics.RetrieveAPIView): allowed_methods = ['GET'] - serializer_class = DocumentVersion + serializer_class = ResourceDocumentVersion queryset = DocumentVersion.objects.all() - class APIDocumentImageView(generics.GenericAPIView): def get(self, request, pk): document = get_object_or_404(Document, pk=pk) diff --git a/mayan/apps/documents/resources.py b/mayan/apps/documents/resources.py index b2962cdaa9..ca8e4defda 100644 --- a/mayan/apps/documents/resources.py +++ b/mayan/apps/documents/resources.py @@ -10,7 +10,6 @@ from .models import Document, DocumentVersion, DocumentPage class ResourceDocumentPage(serializers.HyperlinkedModelSerializer): class Meta: model = DocumentPage - fields = ('url', 'content', 'page_label', 'page_number') class ResourceDocumentVersion(serializers.HyperlinkedModelSerializer): @@ -18,7 +17,6 @@ class ResourceDocumentVersion(serializers.HyperlinkedModelSerializer): class Meta: model = DocumentVersion - fields = ('document', 'major', 'minor', 'micro', 'release_level', 'serial', 'timestamp', 'comment', 'file', 'mimetype', 'encoding', 'filename', 'checksum', 'pages') class ResourceDocument(serializers.HyperlinkedModelSerializer): From 429d8c07eecd937dc3209004ca6a0b204690e4a3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 12:35:53 -0400 Subject: [PATCH 04/29] Initial commit to modernize staging file handling including preview generation --- mayan/apps/main/templates/base.html | 37 +++++--- mayan/apps/sources/__init__.py | 9 +- mayan/apps/sources/api.py | 79 ++++++++++++++++ mayan/apps/sources/classes.py | 60 +++++++++++++ mayan/apps/sources/forms.py | 11 ++- mayan/apps/sources/links.py | 3 +- mayan/apps/sources/models.py | 13 +++ mayan/apps/sources/registry.py | 8 ++ mayan/apps/sources/resources.py | 29 ++++++ mayan/apps/sources/staging.py | 134 ---------------------------- mayan/apps/sources/urls.py | 14 ++- mayan/apps/sources/views.py | 67 +++----------- mayan/apps/sources/widgets.py | 76 ++++++++++++---- 13 files changed, 309 insertions(+), 231 deletions(-) create mode 100644 mayan/apps/sources/api.py create mode 100644 mayan/apps/sources/classes.py create mode 100644 mayan/apps/sources/resources.py delete mode 100644 mayan/apps/sources/staging.py diff --git a/mayan/apps/main/templates/base.html b/mayan/apps/main/templates/base.html index ca3e3cc88d..ff9c7055b0 100644 --- a/mayan/apps/main/templates/base.html +++ b/mayan/apps/main/templates/base.html @@ -143,24 +143,37 @@ }); $("a.fancybox").fancybox({ - openEffect : 'elastic', - closeEffect : 'elastic', + 'openEffect' : 'elastic', + 'closeEffect' : 'elastic', prevEffect : 'none', nextEffect : 'none', 'titleShow' : true, 'type' : 'image', - 'autoResize': true + 'autoResize': true, }); - $("a.fancybox-staging").fancybox({ - openEffect : 'elastic', - closeEffect : 'elastic', - prevEffect : 'none', - nextEffect : 'none', - 'titleShow' : true, - 'type' : 'image', - 'autoResize': true - }); + $("a.fancybox-staging").click(function(e) { + var $this = $(this); + + $.get($this.attr('href'), function( result ) { + if (result.status == 'success') { + $.fancybox.open([ + { + href : result.data, + title : $this.attr('title'), + 'openEffect' : 'elastic', + 'closeEffect' : 'elastic', + prevEffect : 'none', + nextEffect : 'none', + 'titleShow' : true, + 'type' : 'image', + 'autoResize': true, + }, + ]); + } + }) + e.preventDefault(); + }) $("a.fancybox-noscaling").fancybox({ openEffect : 'elastic', diff --git a/mayan/apps/sources/__init__.py b/mayan/apps/sources/__init__.py index 4aabeeb182..f9e36d9604 100644 --- a/mayan/apps/sources/__init__.py +++ b/mayan/apps/sources/__init__.py @@ -7,6 +7,7 @@ from documents.models import Document from navigation.api import register_links, register_model_list_columns from project_setup.api import register_setup +from .classes import StagingFile from .links import (document_create_multiple, document_create_siblings, staging_file_delete, setup_sources, setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list, @@ -16,23 +17,19 @@ from .links import (document_create_multiple, document_create_siblings, upload_version) from .models import (WebForm, StagingFolder, SourceTransformation, WatchFolder) -from .staging import StagingFile from .widgets import staging_file_thumbnail -register_links(StagingFile, [staging_file_delete]) +register_links([StagingFile], [staging_file_delete]) register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete]) -# register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') -# register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(WebForm, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar') -# register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') register_links(StagingFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) @@ -48,7 +45,7 @@ source_views = ['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch register_model_list_columns(StagingFile, [ {'name': _(u'thumbnail'), 'attribute': - encapsulate(lambda x: staging_file_thumbnail(x)) + encapsulate(lambda x: staging_file_thumbnail(x, gallery_name='staging_list', title=x.filename, size='100')) }, ]) diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py new file mode 100644 index 0000000000..415b2d2960 --- /dev/null +++ b/mayan/apps/sources/api.py @@ -0,0 +1,79 @@ +from __future__ import absolute_import + +import logging + +from django.core.exceptions import PermissionDenied +from django.shortcuts import get_object_or_404 + +from rest_framework import generics +from rest_framework.response import Response + +from converter.exceptions import UnkownConvertError, UnknownFileFormat +from converter.literals import DEFAULT_PAGE_NUMBER, DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL +from documents.conf.settings import DISPLAY_SIZE, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL + +#from acls.models import AccessEntry +#from permissions.models import Permission +from .classes import StagingFile +from .resources import SerializerStagingFolder, SerializerStagingFolderFile +from .models import StagingFolder +#from .permissions import PERMISSION_DOCUMENT_VIEW + +logger = logging.getLogger(__name__) + +# API Views + +class APIStagingSourceFileView(generics.GenericAPIView): + def get(self, request, staging_folder_pk, filename): + staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) + return Response(SerializerStagingFolderFile(staging_folder.get_file(filename)).data) + + +class APIStagingSourceListView(generics.ListAPIView): + serializer_class = SerializerStagingFolder + queryset = StagingFolder.objects.all() + + +class APIStagingSourceView(generics.RetrieveAPIView): + serializer_class = SerializerStagingFolder + queryset = StagingFolder.objects.all() + + +class APIStagingSourceFileImageView(generics.GenericAPIView): + def get(self, request, staging_folder_pk, filename): + staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) + + staging_file = staging_folder.get_file(filename) + + #try: + # Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) + #except PermissionDenied: + # AccessEntry.objects.check_access(PERMISSION_DOCUMENT_VIEW, request.user, document) + # Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION]) + + size = request.GET.get('size', DISPLAY_SIZE) + + page = int(request.GET.get('page', DEFAULT_PAGE_NUMBER)) + + zoom = int(request.GET.get('zoom', DEFAULT_ZOOM_LEVEL)) + + if request.GET.get('as_base64', False): + base64_version = True + + if zoom < ZOOM_MIN_LEVEL: + zoom = ZOOM_MIN_LEVEL + + if zoom > ZOOM_MAX_LEVEL: + zoom = ZOOM_MAX_LEVEL + + rotation = int(request.GET.get('rotation', DEFAULT_ROTATION)) % 360 + + try: + return Response({'status': 'success', + 'data': staging_file.get_image(size=size, page=page, zoom=zoom, rotation=rotation, as_base64=True) + }) + except UnknownFileFormat as exception: + return Response({'status': 'error', 'detail': 'unknown_file_format', 'message': unicode(exception)}) + except UnkownConvertError as exception: + return Response({'status': 'error', 'detail': 'converter_error', 'message': unicode(exception)}) + diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py new file mode 100644 index 0000000000..b8fce53c8e --- /dev/null +++ b/mayan/apps/sources/classes.py @@ -0,0 +1,60 @@ +from __future__ import absolute_import + +import base64 +import errno +import os + +from django.core.exceptions import ObjectDoesNotExist +from django.core.files import File +from django.core.files.storage import FileSystemStorage +from django.utils.encoding import smart_str +from django.utils.translation import ugettext + +from converter.api import convert, cache_cleanup +from converter.literals import (DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, + DEFAULT_PAGE_NUMBER, DEFAULT_FILE_FORMAT_MIMETYPE) +from documents.conf.settings import DISPLAY_SIZE, THUMBNAIL_SIZE +from mimetype.api import get_mimetype + + +class StagingFile(object): + """ + Simple class to extend the File class to add preview capabilities + files in a directory on a storage + """ + def __init__(self, staging_folder, filename): + self.staging_folder = staging_folder + self.filename = filename + + def __unicode__(self): + return unicode(self.filename) + + def open(self): + return open(self.get_full_path(), mode='rb') + #return File(file=descriptor, name=filename) + + def get_full_path(self): + return os.path.join(self.staging_folder.folder_path, self.filename) + + def get_image(self, size, page, zoom, rotation, as_base64=True): + #return self.get_valid_image(size=size, transformations=transformations) + converted_file_path = convert(self.get_full_path(), size=size)#, cleanup_files=True)#, transformations=transformations) + + if as_base64: + mimetype = get_mimetype(open(converted_file_path, 'r'), converted_file_path, mimetype_only=True)[0] + image = open(converted_file_path, 'r') + base64_data = base64.b64encode(image.read()) + image.close() + return u'data:%s;base64,%s' % (mimetype, base64_data) + else: + return file_path + + #def delete(self, preview_size, transformations): + # cache_cleanup(self.filepath, size=preview_size, transformations=transformations) + # try: + # os.unlink(self.filepath) + # except OSError, exc: + # if exc.errno == errno.ENOENT: + # pass + # else: + # raise Exception(ugettext(u'Unable to delete staging file: %s') % exc) diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index 714343530b..2263102e30 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +import logging + from django import forms from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext @@ -11,6 +13,8 @@ from .models import (WebForm, StagingFolder, SourceTransformation, from .widgets import FamFamRadioSelect from .utils import validate_whitelist_blacklist +logger = logging.getLogger(__name__) + class StagingDocumentForm(DocumentForm): """ @@ -18,15 +22,16 @@ class StagingDocumentForm(DocumentForm): StagingFile class passed as 'cls' argument """ def __init__(self, *args, **kwargs): - cls = kwargs.pop('cls') show_expand = kwargs.pop('show_expand', False) self.source = kwargs.pop('source') super(StagingDocumentForm, self).__init__(*args, **kwargs) + try: self.fields['staging_file_id'].choices = [ - (staging_file.id, staging_file) for staging_file in cls.get_all() + (hash(staging_file), unicode(staging_file)) for staging_file in self.source.get_files() ] - except: + except Exception as exception: + logger.error('exception: %s' % exception) pass if show_expand: diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index f62dfd9076..38b666f465 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -13,8 +13,7 @@ from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, document_create_multiple = {'text': _(u'upload new documents'), 'view': 'document_create_multiple', 'famfam': 'page_add', 'permissions': [PERMISSION_DOCUMENT_CREATE], 'children_view_regex': [r'upload_interactive']} document_create_siblings = {'text': _(u'clone metadata'), 'view': 'document_create_siblings', 'args': 'object.id', 'famfam': 'page_copy', 'permissions': [PERMISSION_DOCUMENT_CREATE]} -staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} -staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} +staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.pk', 'object.filename'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_']} setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 8c8dcab399..b4ea42f645 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -2,10 +2,12 @@ from __future__ import absolute_import from ast import literal_eval import logging +import os from django.contrib.contenttypes import generic from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError +from django.core.files.storage import FileSystemStorage from django.db import models, transaction from django.utils.translation import ugettext_lazy as _ @@ -20,6 +22,7 @@ from history.api import create_history from metadata.api import save_metadata_list from scheduler.api import register_interval_job, remove_job +from .classes import StagingFile from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, @@ -166,6 +169,16 @@ class StagingFolder(InteractiveBaseModel): return DIMENSION_SEPARATOR.join(dimensions) + def get_file(self, filename): + return StagingFile(staging_folder=self, filename=filename) + + def get_files(self): + try: + for entry in sorted([os.path.normcase(f) for f in os.listdir(self.folder_path) if os.path.isfile(os.path.join(self.folder_path, f))]): + yield self.get_file(filename=entry) + except OSError as exception: + raise Exception(ugettext(u'Unable get list of staging files: %s') % exception) + class Meta(InteractiveBaseModel.Meta): verbose_name = _(u'staging folder') verbose_name_plural = _(u'staging folders') diff --git a/mayan/apps/sources/registry.py b/mayan/apps/sources/registry.py index 520ea4d909..2e4b6bdd54 100644 --- a/mayan/apps/sources/registry.py +++ b/mayan/apps/sources/registry.py @@ -1,6 +1,14 @@ from __future__ import absolute_import +from django.conf.urls import url + from .cleanup import cleanup cleanup_functions = [cleanup] + +#version_0_api_services = [ +# #{'urlpattern': url(r'^staging_file/(?P[0-9]+)/image/$', APIStagingSourceView.as_view(), name='staging-file-image'), 'description': 'Return a base64 image of the staging file', 'url': 'staging_file//image/?page=&zoom=&rotate='}, +# {'urlpattern': url(r'^staging_folder/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='staging-source-list'), 'description': '', 'url': 'staging_file//image/?page=&zoom=&rotate='}, +#] + diff --git a/mayan/apps/sources/resources.py b/mayan/apps/sources/resources.py new file mode 100644 index 0000000000..5728fe92d5 --- /dev/null +++ b/mayan/apps/sources/resources.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import + +from rest_framework import serializers +from rest_framework.reverse import reverse + +from .classes import StagingFile +from .models import StagingFolder + + +class SerializerStagingFolderFile(serializers.Serializer): + url = serializers.SerializerMethodField('get_url') + image_url = serializers.SerializerMethodField('get_image_url') + filename = serializers.CharField(max_length=255) + + def get_url(self, obj): + return reverse('stagingfolderfile-detail', args=[obj.staging_folder.pk, obj.filename], request=self.context.get('request')) + + def get_image_url(self, obj): + return reverse('stagingfolderfile-image-view', args=[obj.staging_folder.pk, obj.filename], request=self.context.get('request')) + + +class SerializerStagingFolder(serializers.HyperlinkedModelSerializer): + files = serializers.SerializerMethodField('get_files') + + def get_files(self, obj): + return [SerializerStagingFolderFile(entry).data for entry in obj.get_files()] + + class Meta: + model = StagingFolder diff --git a/mayan/apps/sources/staging.py b/mayan/apps/sources/staging.py deleted file mode 100644 index da06d140b1..0000000000 --- a/mayan/apps/sources/staging.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import absolute_import - -import errno -import hashlib -import os - -from django.core.exceptions import ObjectDoesNotExist -from django.core.files.base import File -from django.utils.encoding import smart_str -from django.utils.translation import ugettext - -from converter.api import convert, cache_cleanup -from converter.exceptions import UnknownFileFormat, UnkownConvertError -from documents.conf.settings import THUMBNAIL_SIZE -from mimetype.api import (get_icon_file_path, get_error_icon_file_path, - get_mimetype) - - -DEFAULT_STAGING_DIRECTORY = u'/tmp' - -HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() - - -def get_all_files(path): - try: - return sorted([os.path.normcase(f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]) - except OSError, exc: - raise Exception(ugettext(u'Unable get list of staging files: %s') % exc) - - -def _return_new_class(): - return type('StagingFile', (StagingFile,), dict(StagingFile.__dict__)) - - -def create_staging_file_class(request, directory_path, source=None): - cls = _return_new_class() - # cls.set_path(evaluate_user_staging_path(request, source)) - cls.set_path(directory_path) - if source is not None: - cls.set_source(source) - return cls - - -class StagingFile(object): - """ - Simple class to encapsulate the files in a directory and hide the - specifics to the view - """ - path = DEFAULT_STAGING_DIRECTORY - source = None - - @classmethod - def set_path(cls, path): - cls.path = path - - @classmethod - def set_source(cls, source): - cls.source = source - - @classmethod - def get_all(cls): - """ - Return a list of StagingFile instances corresponding to the - current path - """ - staging_files = [] - for filename in get_all_files(cls.path): - staging_files.append(StagingFile( - filepath=os.path.join(cls.path, filename), source=cls.source)) - - return staging_files - - @classmethod - def get(cls, id): - """ - Return a single StagingFile instance corresponding to the id - given as argument - """ - files_dict = dict([(file.id, file) for file in cls.get_all()]) - if id in files_dict: - return files_dict[id] - else: - raise ObjectDoesNotExist - - def __init__(self, filepath, source=None): - self.source = source - self.filepath = filepath - self.filename = os.path.basename(filepath) - self._id = HASH_FUNCTION(smart_str(filepath)) - - def __unicode__(self): - return self.filename - - def __repr__(self): - return self.__unicode__() - - def __getattr__(self, name): - if name == 'id': - return self._id - else: - raise AttributeError - - def upload(self): - """ - Return a StagingFile encapsulated in a File class instance to - allow for easier upload of staging files - """ - try: - return File(file(self.filepath, 'rb'), name=self.filename) - except Exception, exc: - raise Exception(ugettext(u'Unable to upload staging file: %s') % exc) - - def delete(self, preview_size, transformations): - cache_cleanup(self.filepath, size=preview_size, transformations=transformations) - try: - os.unlink(self.filepath) - except OSError, exc: - if exc.errno == errno.ENOENT: - pass - else: - raise Exception(ugettext(u'Unable to delete staging file: %s') % exc) - - def get_valid_image(self, size=THUMBNAIL_SIZE, transformations=None): - return convert(self.filepath, size=size, cleanup_files=False, transformations=transformations) - - def get_image(self, size, transformations): - try: - return self.get_valid_image(size=size, transformations=transformations) - # return convert(self.filepath, size=size, cleanup_files=False, transformations=transformations) - except UnknownFileFormat: - mimetype, encoding = get_mimetype(open(self.filepath, 'rb'), self.filepath) - return get_icon_file_path(mimetype) - except UnkownConvertError: - return get_error_icon_file_path() diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 2936597e10..9487eb1da5 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -2,14 +2,13 @@ from __future__ import absolute_import from django.conf.urls import patterns, url +from .api import APIStagingSourceListView,APIStagingSourceView, APIStagingSourceFileView, APIStagingSourceFileImageView from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH) from .wizards import DocumentCreateWizard urlpatterns = patterns('sources.views', - url(r'^staging_file/type/(?P\w+)/(?P\d+)/(?P\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), - url(r'^staging_file/type/(?P\w+)/(?P\d+)/(?P\w+)/delete/$', 'staging_file_delete', (), 'staging_file_delete'), - url(r'^staging_file/type/staging_folder/(?P\d+)/(?P\w+)/thumbnail/$', 'staging_file_thumbnail', (), 'staging_file_thumbnail'), + url(r'^staging_file/(?P\d+)/(?P[0-9_\.\w-]+)/delete/$', 'staging_file_delete', name='staging_file_delete'), url(r'^upload/document/new/interactive/(?P\w+)/(?P\d+)/$', 'upload_interactive', (), 'upload_interactive'), url(r'^upload/document/new/interactive/$', 'upload_interactive', (), 'upload_interactive'), @@ -37,4 +36,13 @@ urlpatterns = patterns('sources.views', url(r'^create/from/local/multiple/$', DocumentCreateWizard.as_view(), name='document_create_multiple'), url(r'^(?P\d+)/create/siblings/$', 'document_create_siblings', (), 'document_create_siblings'), + + +#version_0_api_services = [ + url(r'^api/staging_folder/file/(?P[0-9]+)/(?P[0-9_\.\w-]+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), + url(r'^api/staging_folder/file/(?P[0-9]+)/(?P[0-9_\.\w-]+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), + url(r'^api/staging_folder/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), + url(r'^api/staging_folder/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') +#] + ) diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 2cc55de94b..7567b318cd 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from django.conf import settings from django.contrib import messages from django.core.exceptions import PermissionDenied +from django.core.files.storage import FileSystemStorage from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect from django.shortcuts import render_to_response, get_object_or_404 @@ -219,10 +220,10 @@ def upload_interactive(request, source_type=None, source_id=None, document_pk=No elif source_type == SOURCE_CHOICE_STAGING: staging_folder = get_object_or_404(StagingFolder, pk=source_id) context['source'] = staging_folder - StagingFile = create_staging_file_class(request, staging_folder.folder_path, source=staging_folder) + if request.method == 'POST': form = StagingDocumentForm(request.POST, request.FILES, - cls=StagingFile, document_type=document_type, + document_type=document_type, show_expand=(staging_folder.uncompress == SOURCE_UNCOMPRESS_CHOICE_ASK) and not document, source=staging_folder, instance=document @@ -280,15 +281,14 @@ def upload_interactive(request, source_type=None, source_id=None, document_pk=No raise messages.error(request, _(u'Unhandled exception: %s') % e) else: - form = StagingDocumentForm(cls=StagingFile, - document_type=document_type, + form = StagingDocumentForm(document_type=document_type, show_expand=(staging_folder.uncompress == SOURCE_UNCOMPRESS_CHOICE_ASK) and not document, source=staging_folder, instance=document ) try: - staging_filelist = StagingFile.get_all() - except Exception, e: + staging_filelist = list(staging_folder.get_files()) + except Exception as e: messages.error(request, e) staging_filelist = [] finally: @@ -374,63 +374,20 @@ def get_form_filename(form): return filename -def staging_file_preview(request, source_type, source_id, staging_file_id): +def staging_file_delete(request, staging_folder_pk, filename): Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION]) - staging_folder = get_object_or_404(StagingFolder, pk=source_id) - StagingFile = create_staging_file_class(request, staging_folder.folder_path) - transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder) + staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) - output_file = StagingFile.get(staging_file_id).get_image( - size=staging_folder.get_preview_size(), - transformations=transformations - ) - if errors and (request.user.is_staff or request.user.is_superuser): - for error in errors: - messages.warning(request, _(u'Staging file transformation error: %(error)s') % { - 'error': error - }) - - return sendfile.sendfile(request, output_file) - - -def staging_file_thumbnail(request, source_id, staging_file_id): - Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION]) - staging_folder = get_object_or_404(StagingFolder, pk=source_id) - StagingFile = create_staging_file_class(request, staging_folder.folder_path, source=staging_folder) - transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder) - - output_file = StagingFile.get(staging_file_id).get_image( - size=THUMBNAIL_SIZE, - transformations=transformations - ) - if errors and (request.user.is_staff or request.user.is_superuser): - for error in errors: - messages.warning(request, _(u'Staging file transformation error: %(error)s') % { - 'error': error - }) - - return sendfile.sendfile(request, output_file) - - -def staging_file_delete(request, source_type, source_id, staging_file_id): - Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION]) - staging_folder = get_object_or_404(StagingFolder, pk=source_id) - StagingFile = create_staging_file_class(request, staging_folder.folder_path) - - staging_file = StagingFile.get(staging_file_id) + staging_file = staging_folder.get_file(filename) next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/'))) previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', '/'))) if request.method == 'POST': try: - transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder) - staging_file.delete( - preview_size=staging_folder.get_preview_size(), - transformations=transformations - ) + staging_file.delete() messages.success(request, _(u'Staging file delete successfully.')) - except Exception, e: - messages.error(request, _(u'Staging file delete error; %s.') % e) + except Exception as exception: + messages.error(request, _(u'Staging file delete error; %s.') % exception) return HttpResponseRedirect(next) results = get_active_tab_links() diff --git a/mayan/apps/sources/widgets.py b/mayan/apps/sources/widgets.py index f55223290a..2d7fdb77a9 100644 --- a/mayan/apps/sources/widgets.py +++ b/mayan/apps/sources/widgets.py @@ -1,9 +1,15 @@ from django import forms -from django.utils.safestring import mark_safe -from django.utils.encoding import force_unicode from django.conf import settings -from django.utils.translation import ugettext_lazy as _ from django.core.urlresolvers import reverse +from django.utils.encoding import force_unicode +from django.utils.html import strip_tags +from django.utils.http import urlencode +from django.utils.safestring import mark_safe +from django.utils.translation import ugettext_lazy as _ + +from converter.literals import DEFAULT_PAGE_NUMBER, DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL +from documents.conf.settings import (THUMBNAIL_SIZE, PREVIEW_SIZE, + DISPLAY_SIZE, MULTIPAGE_PREVIEW_SIZE) class FamFamRadioFieldRenderer(forms.widgets.RadioFieldRenderer): @@ -25,17 +31,55 @@ class FamFamRadioSelect(forms.widgets.RadioSelect): renderer = FamFamRadioFieldRenderer -def staging_file_thumbnail(staging_file): - try: - staging_file.get_valid_image() - template = u'%(string)s' - except: - template = u'%(string)s' +def staging_file_thumbnail(staging_file, **kwargs): + return staging_file_html_widget(staging_file, click_view='stagingfolderfile-image-view', **kwargs) - return mark_safe(template % { - 'url': reverse('staging_file_preview', args=[staging_file.source.source_type, staging_file.source.pk, staging_file.id]), - 'thumbnail': reverse('staging_file_thumbnail', args=[staging_file.source.pk, staging_file.id]), - 'static_url': settings.STATIC_URL, - 'string': _(u'thumbnail'), - 'filename': staging_file.filename - }) + +def staging_file_html_widget(staging_file, click_view=None, page=DEFAULT_PAGE_NUMBER, zoom=DEFAULT_ZOOM_LEVEL, rotation=DEFAULT_ROTATION, gallery_name=None, fancybox_class='fancybox-staging', image_class='lazy-load', title=None, size=THUMBNAIL_SIZE, nolazyload=False): + result = [] + + alt_text = _(u'staging file page image') + + query_dict = { + 'page': page, + 'zoom': zoom, + 'rotation': rotation, + 'size': size, + } + + if gallery_name: + gallery_template = u'rel="%s"' % gallery_name + else: + gallery_template = u'' + + query_string = urlencode(query_dict) + + preview_view = u'%s?%s' % (reverse('stagingfolderfile-image-view', args=[staging_file.staging_folder.pk, staging_file.filename]), query_string) + + plain_template = [] + plain_template.append(u'%s' % (preview_view, alt_text)) + + result.append(u'
' % (staging_file.filename, page if page else DEFAULT_PAGE_NUMBER)) + + if title: + title_template = u'title="%s"' % strip_tags(title) + else: + title_template = u'' + + if click_view: + # TODO: fix this hack + query_dict['size'] = PREVIEW_SIZE + query_string = urlencode(query_dict) + result.append(u'' % (gallery_template, fancybox_class, u'%s?%s' % (reverse(click_view, args=[staging_file.staging_folder.pk, staging_file.filename]), query_string), title_template)) + + if nolazyload: + result.append(u'%s' % (preview_view, alt_text)) + else: + result.append(u'%s' % (image_class, preview_view, settings.STATIC_URL, alt_text)) + result.append(u'' % (preview_view, alt_text)) + + if click_view: + result.append(u'') + result.append(u'
') + + return mark_safe(u''.join(result)) From 0dee7f36409dfdbd9273d72d09ad30fa623c4378 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 12:36:26 -0400 Subject: [PATCH 05/29] Make sure no file handler remain open --- mayan/apps/documents/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index b4cbc5e3ba..52fd1943a7 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -143,8 +143,11 @@ class Document(models.Model): logger.debug('file_path: %s' % file_path) if as_base64: + mimetype = get_mimetype(open(file_path, 'r'), file_path, mimetype_only=True)[0] image = open(file_path, 'r') - return u'data:%s;base64,%s' % (get_mimetype(open(file_path, 'r'), file_path, mimetype_only=True)[0], base64.b64encode(image.read())) + base64_data = base64.b64encode(image.read()) + image.close() + return u'data:%s;base64,%s' % (mimetype, base64_data) else: return file_path From b430f08509d4ab70e347d57d8ba96fd5c3dcd4ea Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 15:32:42 -0400 Subject: [PATCH 06/29] Use URL safe encoded filename --- mayan/apps/sources/api.py | 3 +-- mayan/apps/sources/classes.py | 10 ++++++++-- mayan/apps/sources/links.py | 2 +- mayan/apps/sources/models.py | 6 +++--- mayan/apps/sources/urls.py | 6 +++--- mayan/apps/sources/views.py | 2 +- mayan/apps/sources/widgets.py | 4 ++-- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py index 415b2d2960..e9ed582359 100644 --- a/mayan/apps/sources/api.py +++ b/mayan/apps/sources/api.py @@ -42,8 +42,7 @@ class APIStagingSourceView(generics.RetrieveAPIView): class APIStagingSourceFileImageView(generics.GenericAPIView): def get(self, request, staging_folder_pk, filename): staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) - - staging_file = staging_folder.get_file(filename) + staging_file = staging_folder.get_file(encoded_filename=filename) #try: # Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py index b8fce53c8e..e7700cf3d6 100644 --- a/mayan/apps/sources/classes.py +++ b/mayan/apps/sources/classes.py @@ -3,6 +3,7 @@ from __future__ import absolute_import import base64 import errno import os +import urllib from django.core.exceptions import ObjectDoesNotExist from django.core.files import File @@ -22,9 +23,14 @@ class StagingFile(object): Simple class to extend the File class to add preview capabilities files in a directory on a storage """ - def __init__(self, staging_folder, filename): + def __init__(self, staging_folder, filename=None, encoded_filename=None): self.staging_folder = staging_folder - self.filename = filename + if encoded_filename: + self.encoded_filename = str(encoded_filename) + self.filename = base64.urlsafe_b64decode(urllib.unquote_plus(self.encoded_filename)) + else: + self.filename = filename + self.encoded_filename = base64.urlsafe_b64encode(filename) def __unicode__(self): return unicode(self.filename) diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index 38b666f465..81d5ed40cb 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -13,7 +13,7 @@ from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, document_create_multiple = {'text': _(u'upload new documents'), 'view': 'document_create_multiple', 'famfam': 'page_add', 'permissions': [PERMISSION_DOCUMENT_CREATE], 'children_view_regex': [r'upload_interactive']} document_create_siblings = {'text': _(u'clone metadata'), 'view': 'document_create_siblings', 'args': 'object.id', 'famfam': 'page_copy', 'permissions': [PERMISSION_DOCUMENT_CREATE]} -staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.pk', 'object.filename'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} +staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.pk', 'object.encoded_filename'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_']} setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index b4ea42f645..19cfb418b7 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -169,15 +169,15 @@ class StagingFolder(InteractiveBaseModel): return DIMENSION_SEPARATOR.join(dimensions) - def get_file(self, filename): - return StagingFile(staging_folder=self, filename=filename) + def get_file(self, *args, **kwargs): + return StagingFile(staging_folder=self, *args, **kwargs) def get_files(self): try: for entry in sorted([os.path.normcase(f) for f in os.listdir(self.folder_path) if os.path.isfile(os.path.join(self.folder_path, f))]): yield self.get_file(filename=entry) except OSError as exception: - raise Exception(ugettext(u'Unable get list of staging files: %s') % exception) + raise Exception(_(u'Unable get list of staging files: %s') % exception) class Meta(InteractiveBaseModel.Meta): verbose_name = _(u'staging folder') diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 9487eb1da5..38dc02b561 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -8,7 +8,7 @@ from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, from .wizards import DocumentCreateWizard urlpatterns = patterns('sources.views', - url(r'^staging_file/(?P\d+)/(?P[0-9_\.\w-]+)/delete/$', 'staging_file_delete', name='staging_file_delete'), + url(r'^staging_file/(?P\d+)/(?P.+)/delete/$', 'staging_file_delete', name='staging_file_delete'), url(r'^upload/document/new/interactive/(?P\w+)/(?P\d+)/$', 'upload_interactive', (), 'upload_interactive'), url(r'^upload/document/new/interactive/$', 'upload_interactive', (), 'upload_interactive'), @@ -39,8 +39,8 @@ urlpatterns = patterns('sources.views', #version_0_api_services = [ - url(r'^api/staging_folder/file/(?P[0-9]+)/(?P[0-9_\.\w-]+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), - url(r'^api/staging_folder/file/(?P[0-9]+)/(?P[0-9_\.\w-]+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), + url(r'^api/staging_folder/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), + url(r'^api/staging_folder/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), url(r'^api/staging_folder/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), url(r'^api/staging_folder/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') #] diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 7567b318cd..d7e00ad0bc 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -378,7 +378,7 @@ def staging_file_delete(request, staging_folder_pk, filename): Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION]) staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) - staging_file = staging_folder.get_file(filename) + staging_file = staging_folder.get_file(encoded_filename=filename) next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/'))) previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', '/'))) diff --git a/mayan/apps/sources/widgets.py b/mayan/apps/sources/widgets.py index 2d7fdb77a9..9353500c97 100644 --- a/mayan/apps/sources/widgets.py +++ b/mayan/apps/sources/widgets.py @@ -54,7 +54,7 @@ def staging_file_html_widget(staging_file, click_view=None, page=DEFAULT_PAGE_NU query_string = urlencode(query_dict) - preview_view = u'%s?%s' % (reverse('stagingfolderfile-image-view', args=[staging_file.staging_folder.pk, staging_file.filename]), query_string) + preview_view = u'%s?%s' % (reverse('stagingfolderfile-image-view', args=[staging_file.staging_folder.pk, staging_file.encoded_filename]), query_string) plain_template = [] plain_template.append(u'%s' % (preview_view, alt_text)) @@ -70,7 +70,7 @@ def staging_file_html_widget(staging_file, click_view=None, page=DEFAULT_PAGE_NU # TODO: fix this hack query_dict['size'] = PREVIEW_SIZE query_string = urlencode(query_dict) - result.append(u'' % (gallery_template, fancybox_class, u'%s?%s' % (reverse(click_view, args=[staging_file.staging_folder.pk, staging_file.filename]), query_string), title_template)) + result.append(u'' % (gallery_template, fancybox_class, u'%s?%s' % (reverse(click_view, args=[staging_file.staging_folder.pk, staging_file.encoded_filename]), query_string), title_template)) if nolazyload: result.append(u'%s' % (preview_view, alt_text)) From f911d06823e84fbec48b0dbcee9ef43b51888b9b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 15:42:24 -0400 Subject: [PATCH 07/29] Add the option to suppress or not exceptions during deletion --- mayan/apps/common/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index 706ff199df..0c8d6ec103 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -454,11 +454,14 @@ def load_backend(backend_string): raise -def fs_cleanup(filename): +def fs_cleanup(filename, suppress_exceptions=True): """ Tries to remove the given filename. Ignores non-existent files """ try: os.remove(filename) except OSError: - pass + if suppress_exceptions: + pass + else: + raise From d10aa1e47e296fc890b461c7e03a4001ab8106df Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 15:43:00 -0400 Subject: [PATCH 08/29] Enabled deletion of staging files --- mayan/apps/sources/classes.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py index e7700cf3d6..412c28970b 100644 --- a/mayan/apps/sources/classes.py +++ b/mayan/apps/sources/classes.py @@ -10,7 +10,9 @@ from django.core.files import File from django.core.files.storage import FileSystemStorage from django.utils.encoding import smart_str from django.utils.translation import ugettext +from django.utils.translation import ugettext_lazy as _ +from common.utils import fs_cleanup from converter.api import convert, cache_cleanup from converter.literals import (DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, DEFAULT_PAGE_NUMBER, DEFAULT_FILE_FORMAT_MIMETYPE) @@ -43,8 +45,8 @@ class StagingFile(object): return os.path.join(self.staging_folder.folder_path, self.filename) def get_image(self, size, page, zoom, rotation, as_base64=True): - #return self.get_valid_image(size=size, transformations=transformations) - converted_file_path = convert(self.get_full_path(), size=size)#, cleanup_files=True)#, transformations=transformations) + # TODO: add support for transformations + converted_file_path = convert(self.get_full_path(), size=size) if as_base64: mimetype = get_mimetype(open(converted_file_path, 'r'), converted_file_path, mimetype_only=True)[0] @@ -55,12 +57,5 @@ class StagingFile(object): else: return file_path - #def delete(self, preview_size, transformations): - # cache_cleanup(self.filepath, size=preview_size, transformations=transformations) - # try: - # os.unlink(self.filepath) - # except OSError, exc: - # if exc.errno == errno.ENOENT: - # pass - # else: - # raise Exception(ugettext(u'Unable to delete staging file: %s') % exc) + def delete(self): + os.unlink(self.get_full_path()) From b0783e58163e725a9f5b0cf7ef4756f7274caef6 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 15:54:33 -0400 Subject: [PATCH 09/29] Add support for returning a staging file as a file like object --- mayan/apps/sources/classes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py index 412c28970b..d54c9d2608 100644 --- a/mayan/apps/sources/classes.py +++ b/mayan/apps/sources/classes.py @@ -37,9 +37,8 @@ class StagingFile(object): def __unicode__(self): return unicode(self.filename) - def open(self): - return open(self.get_full_path(), mode='rb') - #return File(file=descriptor, name=filename) + def as_file(self): + return File(file=open(self.get_full_path(), mode='rb'), name=self.filename) def get_full_path(self): return os.path.join(self.staging_folder.folder_path, self.filename) From 5d464058006fc7cfbc011537a30203514c11f8c3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 15:55:05 -0400 Subject: [PATCH 10/29] Fix uploading of staging files --- mayan/apps/sources/forms.py | 2 +- mayan/apps/sources/views.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index 2263102e30..127d13be62 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -28,7 +28,7 @@ class StagingDocumentForm(DocumentForm): try: self.fields['staging_file_id'].choices = [ - (hash(staging_file), unicode(staging_file)) for staging_file in self.source.get_files() + (staging_file.encoded_filename, unicode(staging_file)) for staging_file in self.source.get_files() ] except Exception as exception: logger.error('exception: %s' % exception) diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index d7e00ad0bc..00d1032b1e 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -230,7 +230,7 @@ def upload_interactive(request, source_type=None, source_id=None, document_pk=No ) if form.is_valid(): try: - staging_file = StagingFile.get(form.cleaned_data['staging_file_id']) + staging_file = staging_folder.get_file(encoded_filename=form.cleaned_data['staging_file_id']) if document: expand = False else: @@ -245,7 +245,7 @@ def upload_interactive(request, source_type=None, source_id=None, document_pk=No new_filename = get_form_filename(form) result = staging_folder.upload_file( - staging_file.upload(), + staging_file.as_file(), new_filename, use_file_name=form.cleaned_data.get('use_file_name', False), document_type=document_type, expand=expand, @@ -267,8 +267,7 @@ def upload_interactive(request, source_type=None, source_id=None, document_pk=No messages.warning(request, _(u'Staging file: %s, was not compressed, uploaded as a single file.') % staging_file.filename) if staging_folder.delete_after_upload: - transformations, errors = staging_folder.get_transformation_list() - staging_file.delete(preview_size=staging_folder.get_preview_size(), transformations=transformations) + staging_file.delete() messages.success(request, _(u'Staging file: %s, deleted successfully.') % staging_file.filename) if document: return HttpResponseRedirect(reverse('document_view_simple', args=[document.pk])) From 14977ed860e9794040d21ebef50551b5e85c7fea Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 16:01:16 -0400 Subject: [PATCH 11/29] Return an empty list for files in case of a staging folder exception --- mayan/apps/sources/resources.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mayan/apps/sources/resources.py b/mayan/apps/sources/resources.py index 5728fe92d5..3547343e37 100644 --- a/mayan/apps/sources/resources.py +++ b/mayan/apps/sources/resources.py @@ -1,11 +1,15 @@ from __future__ import absolute_import +import logging + from rest_framework import serializers from rest_framework.reverse import reverse from .classes import StagingFile from .models import StagingFolder +logger = logging.getLogger(__name__) + class SerializerStagingFolderFile(serializers.Serializer): url = serializers.SerializerMethodField('get_url') @@ -23,7 +27,11 @@ class SerializerStagingFolder(serializers.HyperlinkedModelSerializer): files = serializers.SerializerMethodField('get_files') def get_files(self, obj): - return [SerializerStagingFolderFile(entry).data for entry in obj.get_files()] + try: + return [SerializerStagingFolderFile(entry).data for entry in obj.get_files()] + except Exception as exception: + logger.error('unhandled exception: %s' % exception) + return [] class Meta: model = StagingFolder From 93856293933337c6db9771a52fdcc8f25419df5c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 16:02:02 -0400 Subject: [PATCH 12/29] Move REST API app icon to the tools section --- mayan/apps/rest_api/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/rest_api/registry.py b/mayan/apps/rest_api/registry.py index 96b628b40f..83af108688 100644 --- a/mayan/apps/rest_api/registry.py +++ b/mayan/apps/rest_api/registry.py @@ -2,4 +2,4 @@ from __future__ import absolute_import from .links import link_api -setup_links = [link_api] +tool_links = [link_api] From 3ff484a26377617b2e3303feebad23753470b7b7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 16:02:24 -0400 Subject: [PATCH 13/29] Source app API URL minor but important tweak --- mayan/apps/sources/urls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 38dc02b561..a15c01091c 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -39,10 +39,10 @@ urlpatterns = patterns('sources.views', #version_0_api_services = [ - url(r'^api/staging_folder/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), - url(r'^api/staging_folder/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), - url(r'^api/staging_folder/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), - url(r'^api/staging_folder/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') + url(r'^api/staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), + url(r'^api/staging_folders/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), + url(r'^api/staging_folders/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), + url(r'^api/staging_folders/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') #] ) From fc88db50f5b2433a7442ea4cca2eeb77cf90dde6 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 16:16:37 -0400 Subject: [PATCH 14/29] Fix url field of source api endpoints not returning a FQDN --- mayan/apps/sources/api.py | 2 +- mayan/apps/sources/resources.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py index e9ed582359..719e25b870 100644 --- a/mayan/apps/sources/api.py +++ b/mayan/apps/sources/api.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class APIStagingSourceFileView(generics.GenericAPIView): def get(self, request, staging_folder_pk, filename): staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) - return Response(SerializerStagingFolderFile(staging_folder.get_file(filename)).data) + return Response(SerializerStagingFolderFile(staging_folder.get_file(encoded_filename=filename), context={'request': request}).data) class APIStagingSourceListView(generics.ListAPIView): diff --git a/mayan/apps/sources/resources.py b/mayan/apps/sources/resources.py index 3547343e37..020545f467 100644 --- a/mayan/apps/sources/resources.py +++ b/mayan/apps/sources/resources.py @@ -17,10 +17,10 @@ class SerializerStagingFolderFile(serializers.Serializer): filename = serializers.CharField(max_length=255) def get_url(self, obj): - return reverse('stagingfolderfile-detail', args=[obj.staging_folder.pk, obj.filename], request=self.context.get('request')) + return reverse('stagingfolderfile-detail', args=[obj.staging_folder.pk, obj.encoded_filename], request=self.context.get('request')) def get_image_url(self, obj): - return reverse('stagingfolderfile-image-view', args=[obj.staging_folder.pk, obj.filename], request=self.context.get('request')) + return reverse('stagingfolderfile-image-view', args=[obj.staging_folder.pk, obj.encoded_filename], request=self.context.get('request')) class SerializerStagingFolder(serializers.HyperlinkedModelSerializer): @@ -28,7 +28,7 @@ class SerializerStagingFolder(serializers.HyperlinkedModelSerializer): def get_files(self, obj): try: - return [SerializerStagingFolderFile(entry).data for entry in obj.get_files()] + return [SerializerStagingFolderFile(entry, context=self.context).data for entry in obj.get_files()] except Exception as exception: logger.error('unhandled exception: %s' % exception) return [] From 85255207cbdf876132e4239b587987b5e8d600e3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 16:35:27 -0400 Subject: [PATCH 15/29] Cleanups --- mayan/apps/sources/classes.py | 14 ++------------ mayan/apps/sources/models.py | 1 - mayan/apps/sources/resources.py | 1 - mayan/apps/sources/urls.py | 3 ++- mayan/apps/sources/views.py | 5 ----- mayan/apps/sources/widgets.py | 3 +-- 6 files changed, 5 insertions(+), 22 deletions(-) diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py index d54c9d2608..ab04b2d675 100644 --- a/mayan/apps/sources/classes.py +++ b/mayan/apps/sources/classes.py @@ -1,22 +1,12 @@ from __future__ import absolute_import import base64 -import errno import os import urllib -from django.core.exceptions import ObjectDoesNotExist from django.core.files import File -from django.core.files.storage import FileSystemStorage -from django.utils.encoding import smart_str -from django.utils.translation import ugettext -from django.utils.translation import ugettext_lazy as _ -from common.utils import fs_cleanup -from converter.api import convert, cache_cleanup -from converter.literals import (DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, - DEFAULT_PAGE_NUMBER, DEFAULT_FILE_FORMAT_MIMETYPE) -from documents.conf.settings import DISPLAY_SIZE, THUMBNAIL_SIZE +from converter.api import convert from mimetype.api import get_mimetype @@ -54,7 +44,7 @@ class StagingFile(object): image.close() return u'data:%s;base64,%s' % (mimetype, base64_data) else: - return file_path + return converted_file_path def delete(self): os.unlink(self.get_full_path()) diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 19cfb418b7..70b71628e6 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -7,7 +7,6 @@ import os from django.contrib.contenttypes import generic from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError -from django.core.files.storage import FileSystemStorage from django.db import models, transaction from django.utils.translation import ugettext_lazy as _ diff --git a/mayan/apps/sources/resources.py b/mayan/apps/sources/resources.py index 020545f467..abeeee98a0 100644 --- a/mayan/apps/sources/resources.py +++ b/mayan/apps/sources/resources.py @@ -5,7 +5,6 @@ import logging from rest_framework import serializers from rest_framework.reverse import reverse -from .classes import StagingFile from .models import StagingFolder logger = logging.getLogger(__name__) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index a15c01091c..79caaae072 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -2,7 +2,8 @@ from __future__ import absolute_import from django.conf.urls import patterns, url -from .api import APIStagingSourceListView,APIStagingSourceView, APIStagingSourceFileView, APIStagingSourceFileImageView +from .api import (APIStagingSourceListView, APIStagingSourceView, + APIStagingSourceFileView, APIStagingSourceFileImageView) from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH) from .wizards import DocumentCreateWizard diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 00d1032b1e..7cd9bbfadd 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -3,7 +3,6 @@ from __future__ import absolute_import from django.conf import settings from django.contrib import messages from django.core.exceptions import PermissionDenied -from django.core.files.storage import FileSystemStorage from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect from django.shortcuts import render_to_response, get_object_or_404 @@ -13,11 +12,8 @@ from django.utils.safestring import mark_safe from django.utils.translation import ugettext from django.utils.translation import ugettext_lazy as _ -import sendfile - from acls.models import AccessEntry from common.utils import encapsulate -from documents.conf.settings import THUMBNAIL_SIZE from documents.exceptions import NewDocumentVersionNotAllowed from documents.models import DocumentType, Document from documents.permissions import (PERMISSION_DOCUMENT_CREATE, @@ -35,7 +31,6 @@ from .models import (WebForm, StagingFolder, SourceTransformation, from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) -from .staging import create_staging_file_class def document_create_siblings(request, document_id): diff --git a/mayan/apps/sources/widgets.py b/mayan/apps/sources/widgets.py index 9353500c97..e0cbb66fff 100644 --- a/mayan/apps/sources/widgets.py +++ b/mayan/apps/sources/widgets.py @@ -8,8 +8,7 @@ from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from converter.literals import DEFAULT_PAGE_NUMBER, DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL -from documents.conf.settings import (THUMBNAIL_SIZE, PREVIEW_SIZE, - DISPLAY_SIZE, MULTIPAGE_PREVIEW_SIZE) +from documents.conf.settings import THUMBNAIL_SIZE, PREVIEW_SIZE class FamFamRadioFieldRenderer(forms.widgets.RadioFieldRenderer): From 0d0f7be5325a2ee6f3d8946cfed1fba96d66c407 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 17:11:55 -0400 Subject: [PATCH 16/29] Change the way API endpoints are registered --- mayan/apps/rest_api/classes.py | 10 ++++++++++ mayan/apps/rest_api/urls.py | 4 +++- mayan/apps/sources/__init__.py | 7 +++++++ mayan/apps/sources/urls.py | 16 +++++++--------- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/mayan/apps/rest_api/classes.py b/mayan/apps/rest_api/classes.py index b365f4ced5..45bdada1ca 100644 --- a/mayan/apps/rest_api/classes.py +++ b/mayan/apps/rest_api/classes.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +from django.conf.urls import include, patterns, url + class EndPoint(object): _registry = {} @@ -28,3 +30,11 @@ class EndPoint(object): 'urlpattern': urlpattern, } ) + + def register_urls(self, urlpatterns): + from .urls import version_0_endpoints_urlpatterns + endpoint_urls = patterns('', + url(r'^%s/' % self.name, include(urlpatterns)), + ) + + version_0_endpoints_urlpatterns += endpoint_urls diff --git a/mayan/apps/rest_api/urls.py b/mayan/apps/rest_api/urls.py index b0fd73e91f..f1ce1d0d19 100644 --- a/mayan/apps/rest_api/urls.py +++ b/mayan/apps/rest_api/urls.py @@ -2,7 +2,7 @@ from __future__ import absolute_import from django.conf.urls import include, patterns, url -from .classes import EndPoint +#from .classes import EndPoint from .views import APIBase, Version_0, EndPointView version_0_endpoints_urlpatterns = patterns('', @@ -10,6 +10,7 @@ version_0_endpoints_urlpatterns = patterns('', url(r'^(?P\w+)$', EndPointView.as_view(), name='api-version-0-endpoint'), ) +""" for endpoint in EndPoint.get_all(): endpoint_urlpatterns = patterns('') @@ -17,6 +18,7 @@ for endpoint in EndPoint.get_all(): endpoint_urlpatterns += patterns('', service['urlpattern']) version_0_endpoints_urlpatterns += patterns('', url(r'^%s/' % endpoint.name, include(endpoint_urlpatterns))) +""" urlpatterns = patterns('', url(r'^$', APIBase.as_view(), name='api-root'), diff --git a/mayan/apps/sources/__init__.py b/mayan/apps/sources/__init__.py index f9e36d9604..cf146c01a7 100644 --- a/mayan/apps/sources/__init__.py +++ b/mayan/apps/sources/__init__.py @@ -6,7 +6,10 @@ from common.utils import encapsulate from documents.models import Document from navigation.api import register_links, register_model_list_columns from project_setup.api import register_setup +from rest_api.classes import EndPoint +from .api import (APIStagingSourceListView, APIStagingSourceView, + APIStagingSourceFileView, APIStagingSourceFileImageView) from .classes import StagingFile from .links import (document_create_multiple, document_create_siblings, staging_file_delete, setup_sources, setup_web_form_list, @@ -17,6 +20,7 @@ from .links import (document_create_multiple, document_create_siblings, upload_version) from .models import (WebForm, StagingFolder, SourceTransformation, WatchFolder) +from .urls import api_urls from .widgets import staging_file_thumbnail register_links([StagingFile], [staging_file_delete]) @@ -53,3 +57,6 @@ register_setup(setup_sources) register_links([Document, 'document_list_recent', 'document_list', 'document_create', 'document_create_multiple', 'upload_interactive', 'staging_file_delete'], [document_create_multiple], menu_name='secondary_menu') register_links(Document, [document_create_siblings]) + +endpoint = EndPoint('sources') +endpoint.register_urls(api_urls) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 79caaae072..d289bc3b64 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -37,13 +37,11 @@ urlpatterns = patterns('sources.views', url(r'^create/from/local/multiple/$', DocumentCreateWizard.as_view(), name='document_create_multiple'), url(r'^(?P\d+)/create/siblings/$', 'document_create_siblings', (), 'document_create_siblings'), - - -#version_0_api_services = [ - url(r'^api/staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), - url(r'^api/staging_folders/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), - url(r'^api/staging_folders/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), - url(r'^api/staging_folders/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') -#] - +) + +api_urls = patterns('', + url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), + url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), + url(r'^staging_folders/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), + url(r'^staging_folders/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') ) From c4174f8020a4688208633f95f02b5f33de445cf5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 18:50:37 -0400 Subject: [PATCH 17/29] Change the definitions of the per app API properties --- mayan/apps/rest_api/classes.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mayan/apps/rest_api/classes.py b/mayan/apps/rest_api/classes.py index 45bdada1ca..4bb3904210 100644 --- a/mayan/apps/rest_api/classes.py +++ b/mayan/apps/rest_api/classes.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from django.conf.urls import include, patterns, url -class EndPoint(object): +class APIEndPoint(object): _registry = {} @classmethod @@ -19,22 +19,21 @@ class EndPoint(object): def __init__(self, name): self.name = name - self.services = [] + self.endpoints = [] self.__class__._registry[name] = self - def add_service(self, urlpattern, url=None, description=None): - self.services.append( + def add_endpoint(self, view_name, description=None): + self.endpoints.append( { 'description': description, - 'url': url, - 'urlpattern': urlpattern, + 'view_name': view_name, } ) def register_urls(self, urlpatterns): - from .urls import version_0_endpoints_urlpatterns + from .urls import version_0_urlpatterns endpoint_urls = patterns('', url(r'^%s/' % self.name, include(urlpatterns)), ) - version_0_endpoints_urlpatterns += endpoint_urls + version_0_urlpatterns += endpoint_urls From 909dd582df78ceedef2b0285054fc030d30205fe Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 18:51:39 -0400 Subject: [PATCH 18/29] Update the app API class methods, register the sources app main endpoint, add API doc strings --- mayan/apps/app_registry/models.py | 6 +++--- mayan/apps/rest_api/urls.py | 19 ++++------------- mayan/apps/rest_api/views.py | 34 +++++++++++++++++++++---------- mayan/apps/sources/__init__.py | 7 +++---- mayan/apps/sources/api.py | 21 +++++++++++++++---- mayan/apps/sources/urls.py | 12 +++++------ 6 files changed, 56 insertions(+), 43 deletions(-) diff --git a/mayan/apps/app_registry/models.py b/mayan/apps/app_registry/models.py index 45a8a79aca..b0ec4c970d 100644 --- a/mayan/apps/app_registry/models.py +++ b/mayan/apps/app_registry/models.py @@ -9,7 +9,7 @@ from bootstrap.classes import BootstrapModel, Cleanup from navigation.api import register_top_menu from project_setup.api import register_setup from project_tools.api import register_tool -from rest_api.classes import EndPoint +from rest_api.classes import APIEndPoint logger = logging.getLogger(__name__) @@ -60,10 +60,10 @@ class App(object): logger.debug('version_0_api_services: %s' % version_0_api_services) if version_0_api_services: - endpoint = EndPoint(app_name) + api_endpoint = APIEndPoint(app_name) for service in version_0_api_services: - endpoint.add_service(**service) + api_endpoint.add_service(**service) def __unicode__(self): return unicode(self.label) diff --git a/mayan/apps/rest_api/urls.py b/mayan/apps/rest_api/urls.py index f1ce1d0d19..2e22f0d559 100644 --- a/mayan/apps/rest_api/urls.py +++ b/mayan/apps/rest_api/urls.py @@ -2,25 +2,14 @@ from __future__ import absolute_import from django.conf.urls import include, patterns, url -#from .classes import EndPoint -from .views import APIBase, Version_0, EndPointView +from .views import APIBase, Version_0, APIAppView -version_0_endpoints_urlpatterns = patterns('', +version_0_urlpatterns = patterns('', url(r'^$', Version_0.as_view(), name='api-version-0'), - url(r'^(?P\w+)$', EndPointView.as_view(), name='api-version-0-endpoint'), + url(r'^(?P\w+)$', APIAppView.as_view(), name='api-version-0-app'), ) -""" -for endpoint in EndPoint.get_all(): - endpoint_urlpatterns = patterns('') - - for service in endpoint.services: - endpoint_urlpatterns += patterns('', service['urlpattern']) - - version_0_endpoints_urlpatterns += patterns('', url(r'^%s/' % endpoint.name, include(endpoint_urlpatterns))) -""" - urlpatterns = patterns('', url(r'^$', APIBase.as_view(), name='api-root'), - url(r'^v0/', include(version_0_endpoints_urlpatterns)), + url(r'^v0/', include(version_0_urlpatterns)), ) diff --git a/mayan/apps/rest_api/views.py b/mayan/apps/rest_api/views.py index b7920b5693..27a1183a30 100644 --- a/mayan/apps/rest_api/views.py +++ b/mayan/apps/rest_api/views.py @@ -10,7 +10,7 @@ from rest_framework.generics import RetrieveAPIView from rest_framework.response import Response from rest_framework.reverse import reverse -from .classes import EndPoint +from .classes import APIEndPoint logger = logging.getLogger(__name__) @@ -19,6 +19,10 @@ registered_version_0_endpoints = [ class APIBase(generics.GenericAPIView): + """ + Main entry point of the API. + """ + def get(self, request, format=None): return Response([ {'name': 'Version 0', 'url': reverse('api-version-0', request=request, format=format)} @@ -26,27 +30,35 @@ class APIBase(generics.GenericAPIView): class Version_0(generics.GenericAPIView): + """ + API version 0 entry points. + """ + def get(self, request, format=None): return Response({ - 'endpoints': [ - {'name': unicode(endpoint), 'url': reverse('api-version-0-endpoint', args=[unicode(endpoint)], request=request, format=format)} for endpoint in EndPoint.get_all() + 'apps': [ + {'name': unicode(endpoint), 'url': reverse('api-version-0-app', args=[unicode(endpoint)], request=request, format=format)} for endpoint in APIEndPoint.get_all() ], }) -class EndPointView(generics.GenericAPIView): - def get(self, request, endpoint_name, format=None): + +class APIAppView(generics.GenericAPIView): + """ + Entry points of the selected app. + """ + + def get(self, request, app_name, format=None): result = [] - endpoint = EndPoint.get(endpoint_name) - for service in endpoint.services: + api_app = APIEndPoint.get(app_name) + for endpoint in api_app.endpoints: result.append( { - 'description': service['description'], - 'name': service['urlpattern'].name, - 'url': service['url'], + 'description': endpoint['description'], + 'url': reverse(endpoint['view_name'], request=request, format=format), } ) return Response({ - 'services': result + 'endpoints': result }) diff --git a/mayan/apps/sources/__init__.py b/mayan/apps/sources/__init__.py index cf146c01a7..f05e5b45c7 100644 --- a/mayan/apps/sources/__init__.py +++ b/mayan/apps/sources/__init__.py @@ -6,10 +6,8 @@ from common.utils import encapsulate from documents.models import Document from navigation.api import register_links, register_model_list_columns from project_setup.api import register_setup -from rest_api.classes import EndPoint +from rest_api.classes import APIEndPoint -from .api import (APIStagingSourceListView, APIStagingSourceView, - APIStagingSourceFileView, APIStagingSourceFileImageView) from .classes import StagingFile from .links import (document_create_multiple, document_create_siblings, staging_file_delete, setup_sources, setup_web_form_list, @@ -58,5 +56,6 @@ register_setup(setup_sources) register_links([Document, 'document_list_recent', 'document_list', 'document_create', 'document_create_multiple', 'upload_interactive', 'staging_file_delete'], [document_create_multiple], menu_name='secondary_menu') register_links(Document, [document_create_siblings]) -endpoint = EndPoint('sources') +endpoint = APIEndPoint('sources') endpoint.register_urls(api_urls) +endpoint.add_endpoint('stagingfolder-list') diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py index 719e25b870..cd20fefeeb 100644 --- a/mayan/apps/sources/api.py +++ b/mayan/apps/sources/api.py @@ -23,23 +23,36 @@ logger = logging.getLogger(__name__) # API Views -class APIStagingSourceFileView(generics.GenericAPIView): +class StagingSourceFileView(generics.GenericAPIView): + """ + Details of the selected staging file. + """ def get(self, request, staging_folder_pk, filename): staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) return Response(SerializerStagingFolderFile(staging_folder.get_file(encoded_filename=filename), context={'request': request}).data) -class APIStagingSourceListView(generics.ListAPIView): +class StagingSourceListView(generics.ListAPIView): + """ + Returns a list of all the staging folders and the files they contain. + """ + serializer_class = SerializerStagingFolder queryset = StagingFolder.objects.all() -class APIStagingSourceView(generics.RetrieveAPIView): +class StagingSourceView(generics.RetrieveAPIView): + """ + Details of the selected staging folders and the files it contains. + """ serializer_class = SerializerStagingFolder queryset = StagingFolder.objects.all() -class APIStagingSourceFileImageView(generics.GenericAPIView): +class StagingSourceFileImageView(generics.GenericAPIView): + """ + Image of the selected staging file. + """ def get(self, request, staging_folder_pk, filename): staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) staging_file = staging_folder.get_file(encoded_filename=filename) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index d289bc3b64..7be35a5225 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -2,8 +2,8 @@ from __future__ import absolute_import from django.conf.urls import patterns, url -from .api import (APIStagingSourceListView, APIStagingSourceView, - APIStagingSourceFileView, APIStagingSourceFileImageView) +from .api import (StagingSourceListView, StagingSourceView, + StagingSourceFileView, StagingSourceFileImageView) from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH) from .wizards import DocumentCreateWizard @@ -40,8 +40,8 @@ urlpatterns = patterns('sources.views', ) api_urls = patterns('', - url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), - url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), - url(r'^staging_folders/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), - url(r'^staging_folders/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') + url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', StagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), + url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/$', StagingSourceFileView.as_view(), name='stagingfolderfile-detail'), + url(r'^staging_folders/$', StagingSourceListView.as_view(), name='stagingfolder-list'), + url(r'^staging_folders/(?P[0-9]+)/$', StagingSourceView.as_view(), name='stagingfolder-detail') ) From 2396d7dae99ee43acb962bbb7dffc0a632b46977 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 18:52:35 -0400 Subject: [PATCH 19/29] Update the documents app to use the new way of registering API endpoints --- mayan/apps/documents/__init__.py | 7 +++++++ mayan/apps/documents/api.py | 33 +++++++++++++++++++++++++++---- mayan/apps/documents/registry.py | 7 ------- mayan/apps/documents/resources.py | 10 +++++----- mayan/apps/documents/urls.py | 10 +++++++++- 5 files changed, 50 insertions(+), 17 deletions(-) diff --git a/mayan/apps/documents/__init__.py b/mayan/apps/documents/__init__.py index bc5822b6c2..730fbb688b 100644 --- a/mayan/apps/documents/__init__.py +++ b/mayan/apps/documents/__init__.py @@ -14,6 +14,7 @@ from navigation.api import (register_links, register_top_menu, register_model_list_columns, register_multi_item_links, register_sidebar_template) from project_setup.api import register_setup +from rest_api.classes import APIEndPoint from statistics.classes import StatisticNamespace from .conf import settings as document_settings @@ -46,6 +47,7 @@ from .permissions import ( PERMISSION_DOCUMENT_TRANSFORM, PERMISSION_DOCUMENT_EDIT, PERMISSION_DOCUMENT_VERSION_REVERT, PERMISSION_DOCUMENT_NEW_VERSION) from .statistics import DocumentStatistics, DocumentUsageStatistics +from .urls import api_urls from .widgets import document_thumbnail # History setup @@ -151,3 +153,8 @@ document_search.add_related_field('comments', 'Comment', 'comment', 'object_pk', namespace = StatisticNamespace(name='documents', label=_(u'Documents')) namespace.add_statistic(DocumentStatistics(name='document_stats', label=_(u'Document tendencies'))) namespace.add_statistic(DocumentUsageStatistics(name='document_usage', label=_(u'Document usage'))) + + +endpoint = APIEndPoint('documents') +endpoint.register_urls(api_urls) +endpoint.add_endpoint('document-list') diff --git a/mayan/apps/documents/api.py b/mayan/apps/documents/api.py index eccb7ff2f5..c3726cf683 100644 --- a/mayan/apps/documents/api.py +++ b/mayan/apps/documents/api.py @@ -17,32 +17,57 @@ from permissions.models import Permission from .conf.settings import DISPLAY_SIZE, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL from .models import Document, DocumentVersion, DocumentPage from .permissions import PERMISSION_DOCUMENT_VIEW -from .resources import ResourceDocument, ResourceDocumentVersion, ResourceDocumentPage +from .resources import DocumentSerializer, DocumentVersionSerializer, DocumentPageSerializer logger = logging.getLogger(__name__) # API Views +class APIDocumentListView(generics.ListAPIView): + """ + Returns a list of all the documents. + """ + + serializer_class = DocumentSerializer + queryset = Document.objects.all() + + class APIDocumentPageView(generics.RetrieveAPIView): + """ + Returns the selected document page details. + """ + allowed_methods = ['GET'] - serializer_class = ResourceDocumentPage + serializer_class = DocumentPageSerializer queryset = DocumentPage.objects.all() class APIDocumentView(generics.RetrieveAPIView): + """ + Returns the selected document details. + """ + allowed_methods = ['GET'] - serializer_class = ResourceDocument + serializer_class = DocumentSerializer queryset = Document.objects.all() class APIDocumentVersionView(generics.RetrieveAPIView): + """ + Returns the selected document version details. + """ + allowed_methods = ['GET'] - serializer_class = ResourceDocumentVersion + serializer_class = DocumentVersionSerializer queryset = DocumentVersion.objects.all() class APIDocumentImageView(generics.GenericAPIView): + """ + Returns an image representation of the selected document. + """ + def get(self, request, pk): document = get_object_or_404(Document, pk=pk) diff --git a/mayan/apps/documents/registry.py b/mayan/apps/documents/registry.py index ee779b8922..2b7ea97e78 100644 --- a/mayan/apps/documents/registry.py +++ b/mayan/apps/documents/registry.py @@ -3,7 +3,6 @@ from __future__ import absolute_import from django.conf.urls import url from .cleanup import cleanup -from .api import APIDocumentView, APIDocumentVersionView, APIDocumentImageView, APIDocumentPageView bootstrap_models = [ { @@ -16,9 +15,3 @@ bootstrap_models = [ ] cleanup_functions = [cleanup] -version_0_api_services = [ - {'urlpattern': url(r'^document/(?P[0-9]+)/$', APIDocumentView.as_view(), name='document-detail'), 'description': 'Show document data', 'url': 'document/'}, - {'urlpattern': url(r'^document_version/(?P[0-9]+)/$', APIDocumentVersionView.as_view(), name='documentversion-detail'), 'description': '', 'url': 'document_version/'}, - {'urlpattern': url(r'^document_page/(?P[0-9]+)/$', APIDocumentPageView.as_view(), name='documentpage-detail'), 'description': '', 'url': 'document_page/'}, - {'urlpattern': url(r'^document/(?P[0-9]+)/image/$', APIDocumentImageView.as_view(), name='document-image'), 'description': 'Return a base64 image of the document', 'url': 'document//image/?page=&zoom=&rotate='}, -] diff --git a/mayan/apps/documents/resources.py b/mayan/apps/documents/resources.py index ca8e4defda..e8710a488b 100644 --- a/mayan/apps/documents/resources.py +++ b/mayan/apps/documents/resources.py @@ -7,20 +7,20 @@ from rest_framework import serializers from .models import Document, DocumentVersion, DocumentPage -class ResourceDocumentPage(serializers.HyperlinkedModelSerializer): +class DocumentPageSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = DocumentPage -class ResourceDocumentVersion(serializers.HyperlinkedModelSerializer): - pages = ResourceDocumentPage(many=True, read_only=True) +class DocumentVersionSerializer(serializers.HyperlinkedModelSerializer): + pages = DocumentPageSerializer(many=True, read_only=True) class Meta: model = DocumentVersion -class ResourceDocument(serializers.HyperlinkedModelSerializer): - versions = ResourceDocumentVersion(many=True, read_only=True) +class DocumentSerializer(serializers.HyperlinkedModelSerializer): + versions = DocumentVersionSerializer(many=True, read_only=True) class Meta: model = Document diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 06aea0cf7d..9e0848f401 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from django.conf.urls import patterns, url +from .api import APIDocumentView, APIDocumentListView, APIDocumentVersionView, APIDocumentImageView, APIDocumentPageView from .conf.settings import (PREVIEW_SIZE, PRINT_SIZE, DISPLAY_SIZE, MULTIPAGE_PREVIEW_SIZE) @@ -67,5 +68,12 @@ urlpatterns = patterns('documents.views', url(r'^type/filename/(?P\d+)/edit/$', 'document_type_filename_edit', (), 'document_type_filename_edit'), url(r'^type/filename/(?P\d+)/delete/$', 'document_type_filename_delete', (), 'document_type_filename_delete'), url(r'^type/(?P\d+)/filename/create/$', 'document_type_filename_create', (), 'document_type_filename_create'), - +) + +api_urls = patterns('', + url(r'^documents/$', APIDocumentListView.as_view(), name='document-list'), + url(r'^documents/(?P[0-9]+)/$', APIDocumentView.as_view(), name='document-detail'), + url(r'^document_version/(?P[0-9]+)/$', APIDocumentVersionView.as_view(), name='documentversion-detail'), + url(r'^document_page/(?P[0-9]+)/$', APIDocumentPageView.as_view(), name='documentpage-detail'), + url(r'^documents/(?P[0-9]+)/image/$', APIDocumentImageView.as_view(), name='document-image'), ) From 308099c0c2db93e3316363ea5f9212277761df90 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 19:51:12 -0400 Subject: [PATCH 20/29] Prepend API to the API views --- mayan/apps/sources/api.py | 8 ++++---- mayan/apps/sources/urls.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py index cd20fefeeb..78818aba37 100644 --- a/mayan/apps/sources/api.py +++ b/mayan/apps/sources/api.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) # API Views -class StagingSourceFileView(generics.GenericAPIView): +class APIStagingSourceFileView(generics.GenericAPIView): """ Details of the selected staging file. """ @@ -32,7 +32,7 @@ class StagingSourceFileView(generics.GenericAPIView): return Response(SerializerStagingFolderFile(staging_folder.get_file(encoded_filename=filename), context={'request': request}).data) -class StagingSourceListView(generics.ListAPIView): +class APIStagingSourceListView(generics.ListAPIView): """ Returns a list of all the staging folders and the files they contain. """ @@ -41,7 +41,7 @@ class StagingSourceListView(generics.ListAPIView): queryset = StagingFolder.objects.all() -class StagingSourceView(generics.RetrieveAPIView): +class APIStagingSourceView(generics.RetrieveAPIView): """ Details of the selected staging folders and the files it contains. """ @@ -49,7 +49,7 @@ class StagingSourceView(generics.RetrieveAPIView): queryset = StagingFolder.objects.all() -class StagingSourceFileImageView(generics.GenericAPIView): +class APIStagingSourceFileImageView(generics.GenericAPIView): """ Image of the selected staging file. """ diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 7be35a5225..d289bc3b64 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -2,8 +2,8 @@ from __future__ import absolute_import from django.conf.urls import patterns, url -from .api import (StagingSourceListView, StagingSourceView, - StagingSourceFileView, StagingSourceFileImageView) +from .api import (APIStagingSourceListView, APIStagingSourceView, + APIStagingSourceFileView, APIStagingSourceFileImageView) from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH) from .wizards import DocumentCreateWizard @@ -40,8 +40,8 @@ urlpatterns = patterns('sources.views', ) api_urls = patterns('', - url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', StagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), - url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/$', StagingSourceFileView.as_view(), name='stagingfolderfile-detail'), - url(r'^staging_folders/$', StagingSourceListView.as_view(), name='stagingfolder-list'), - url(r'^staging_folders/(?P[0-9]+)/$', StagingSourceView.as_view(), name='stagingfolder-detail') + url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), + url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), + url(r'^staging_folders/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), + url(r'^staging_folders/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='stagingfolder-detail') ) From a62e8a300173449c215acf543bc74ca1ebc897ee Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:17:17 -0400 Subject: [PATCH 21/29] Simplified installation instructions --- docs/intro/installation.rst | 58 +++++++++++++++---------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/docs/intro/installation.rst b/docs/intro/installation.rst index b050511f21..f30a0fa7c2 100644 --- a/docs/intro/installation.rst +++ b/docs/intro/installation.rst @@ -17,63 +17,53 @@ If using a Fedora_ based Linux distribution get the executable requirements usin To initialize a ``virtualenv`` to deploy the project do:: - $ virtualenv --no-site-packages mayan + $ virtualenv --no-site-packages venv + $ source venv/bin/activate Download_ and decompress the latest version of **Mayan EDMS**:: - $ cd mayan + $ cd venv $ tar -xvzf mayan.tar.gz Or clone the latest development version straight from github:: - $ cd mayan - $ git clone git://github.com/mayan-edms/mayan-edms.git + $ cd venv + $ git clone https://github.com/mayan-edms/mayan-edms.git To install the python dependencies ``easy_install`` can be used, however for easier retrieval a production dependencies file is included, to use it execute:: - $ cd mayan - $ source ../bin/activate + $ cd mayan-edms $ pip install -r requirements.txt -Create the database that will hold the data. Install any corresponding python database drivers. Update the settings.py file with you database settings. -If using the ``MySQL`` database manager, use the following commands:: - - $ sudo apt-get install libmysqlclient-dev -y - $ pip install MySQL-python - -If using ``PostgreSQL``, enter the following:: - - $ sudo apt-get install libpq-dev -y - $ pip install psycopg2 - -For Fedora systems just use Yum instead of APT:: - - $ sudo yum install -y mysql-devel - $ pip install MySQL-python - -If using ``PostgreSQL``, enter the following:: - - $ sudo yum install -y postgresql-devel - $ pip install psycopg2 - +By default Mayan EDMS will create a single file SQLite database which makes is very easy to start using Mayan EDMS. Populate the database with the project's schema doing:: $ ./manage.py syncdb --migrate --noinput + To test your installation, execute Django’s development server using the ``runserver`` command to launch a local instance of Mayan EDMS:: $ ./manage.py runserver Point your browser to http://127:0.0.1:8000, if everything was installed -correctly you should see the login screen. After making sure everything -is running correctly, stop the runserver command and delete the settings_local.py. -Deploy **Mayan EDMS** using the webserver of your preference. If your are -using Apache_, a sample site file is included under the contrib directory. +correctly you should see the login screen and panel showing a randomly generated admin password. -Before finally deploying to your favorite webserver don't forget to collect the -static files of the project into the ``static`` folder for serving via a webserver:: - $ ./manage.py collectstatic +Production use +-------------- + +After making sure everything is running correctly, stop the runserver command. +Deploy **Mayan EDMS** using the webserver of your preference. For more information +on deployment instructions and examples checkout Django's official documentation +on the topic https://docs.djangoproject.com/en/1.6/howto/deployment/ + + +Other database managers +----------------------- + +If you want to use a database manager other than SQLite3 install any +corresponding python database drivers and create a settings_local.py file +with the corresponding database settings as shown here: https://docs.djangoproject.com/en/1.6/ref/settings/#std:setting-DATABASES Cloud install From 410731cea69da0bb5bff9ca72c9565e9e0e9a166 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:25:44 -0400 Subject: [PATCH 22/29] Removed 3rd party installation methods --- docs/intro/installation.rst | 134 ++---------------------------------- 1 file changed, 6 insertions(+), 128 deletions(-) diff --git a/docs/intro/installation.rst b/docs/intro/installation.rst index f30a0fa7c2..99aa762589 100644 --- a/docs/intro/installation.rst +++ b/docs/intro/installation.rst @@ -5,7 +5,7 @@ Installation Ubuntu, Debian or Fedora server ------------------------------- -**Mayan EDMS** should be deployed_ like any other Django_ project and preferably using virtualenv_. +**Mayan EDMS** should be deployed like any other Django_ project and preferably using virtualenv_. If using a Debian_ or Ubuntu_ based Linux distribution getting the executable requirements is as easy as:: @@ -15,7 +15,7 @@ If using a Fedora_ based Linux distribution get the executable requirements usin $ sudo yum install -y git gcc tesseract unpaper python-virtualenv ghostscript libjpeg-turbo-devel libpng-devel poppler-util python-devel -To initialize a ``virtualenv`` to deploy the project do:: +Initialize a ``virtualenv`` to deploy the project:: $ virtualenv --no-site-packages venv $ source venv/bin/activate @@ -35,13 +35,13 @@ To install the python dependencies ``easy_install`` can be used, however for eas $ cd mayan-edms $ pip install -r requirements.txt -By default Mayan EDMS will create a single file SQLite database which makes is very easy to start using Mayan EDMS. +By default **Mayan EDMS** will create a single file SQLite_ database which makes is very easy to start using **Mayan EDMS**. Populate the database with the project's schema doing:: $ ./manage.py syncdb --migrate --noinput -To test your installation, execute Django’s development server using the ``runserver`` command to launch a local instance of Mayan EDMS:: +To test your installation, execute Django’s development server using the ``runserver`` command to launch a local instance of **Mayan EDMS**:: $ ./manage.py runserver @@ -61,143 +61,21 @@ on the topic https://docs.djangoproject.com/en/1.6/howto/deployment/ Other database managers ----------------------- -If you want to use a database manager other than SQLite3 install any +If you want to use a database manager other than SQLite_ install any corresponding python database drivers and create a settings_local.py file with the corresponding database settings as shown here: https://docs.djangoproject.com/en/1.6/ref/settings/#std:setting-DATABASES -Cloud install -------------- -SaaS provied Appsembler_ has started providing a "1-click install" cloud -offering of **Mayan EDMS**. Go to their website and click on apps to start -your trial period of **Mayan EDMS** on the cloud. - -Webfaction ----------- - -To install **Mayan EDMS** on Webfaction_, follow these steps: - -1. Create a new database: - - * Enter the following selections: - - * Type:* ``Mysql`` - * Name:* ``_mayan`` - * Encoding:* ``utf-8`` - - * Anotate the provided password. - -2. Create a new app: - - * Enter the following in the textbox: - - * Name:* ``mayan_app`` - * App category:* ``mod_wsgi`` - * App type:* ``mod_wsgi 3.3/Python 2.7`` - -3. Login via ssh, and execute:: - - $ easy_install-2.7 virtualenv - $ cd ~/webapps/mayan_app - $ virtualenv --no-site-packages mayan - $ cd mayan - $ git clone git://github.com/mayan-edms/mayan-edms.git - $ cd mayan - $ source ../bin/activate - $ pip install -r requirements.txt - -4. Install the Python MySQL database driver:: - - $ pip install MySQL-python - -5. Create a settings_local.py file, and paste into it the following:: - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': '_mayan', - 'USER': '_mayan', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '', - } - } - -6. Create the database schema:: - - $ ./manage.py syncdb --migrate --noinput - -7. Collect the static files of the apps:: - - $ ./manage.py collectstatic -l --noinput - -8. Create a new app: - - * Enter the following: - - * Name:* ``mayan_static`` - * App category:* ``Symbolic link`` - * App type:* ``Symbolic link to static-only app`` - * Extra info: ``/home//webapps/mayan_app/mayan/mayan/static`` - -9. Create the website: - - * Name: ``mayan_edms`` - * Choose a subdomain - * Under ``Site apps:`` enter the following selections: - - * App #1 - - * App:* ``mayan_app`` - * URL path (ex: '/' or '/blog'):* ``/`` - - * App #2 - - * App:* ``mayan_static`` - * URL path (ex: '/' or '/blog'):* ``/mayan-static`` - -10. Edit the file ``~/webapps/mayan_app/apache2/conf/httpd.conf``: - - * Disable the ``DirectoryIndex`` line and the ``DocumentRoot`` line. - * Add the following line:: - - WSGIScriptAlias / /home//webapps/mayan_app/mayan/mayan/wsgi/dispatch.wsgi - - * Tune your WSGI process to only use 2 workers (as explained here: `Reducing mod_wsgi Memory Consumption`_) - to keep the memory usage under the basic 256MB of RAM provided or upgrade your plan to 512MB, - the line that controls the amount of workers launched is:: - - WSGIDaemonProcess mayan_app processes=5 python-path=/home//webapps/mayan_app/lib/python2.7 threads=1 - - change it to:: - - WSGIDaemonProcess mayan_app processes=2 python-path=/home//webapps/mayan_app/lib/python2.7 threads=1 - - -11. Restart your apache instance: - - * Execute:: - - apache2/bin/restart - - .. _`vendor lock-in`: https://secure.wikimedia.org/wikipedia/en/wiki/Vendor_lock-in .. _Python: http://www.python.org/ .. _Django: http://www.djangoproject.com/ .. _OCR: https://secure.wikimedia.org/wikipedia/en/wiki/Optical_character_recognition .. _`Open source`: https://secure.wikimedia.org/wikipedia/en/wiki/Open_source -.. _DjangoZoom: http://djangozoom.com/ -.. _Youtube: http://bit.ly/mayan-djangozoom .. _Django: http://www.djangoproject.com/ - - .. _Apache: https://www.apache.org/ .. _Debian: http://www.debian.org/ .. _Ubuntu: http://www.ubuntu.com/ .. _Download: https://github.com/mayan-edms/mayan-edms/archives/master -.. _Webfaction: http://www.webfaction.com -.. _deployed: https://docs.djangoproject.com/en/1.3/howto/deployment/ .. _virtualenv: http://www.virtualenv.org/en/latest/index.html -.. _`Reducing mod_wsgi Memory Consumption`: http://docs.webfaction.com/software/mod-wsgi.html#mod-wsgi-reducing-memory-consumption .. _Fedora: http://fedoraproject.org/ -.. _Appsembler: http://appsembler.com/ +.. _SQLite: https://www.sqlite.org/ From 494559cdaab77c139650e180f3b03e8946dbf251 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:33:06 -0400 Subject: [PATCH 23/29] Break the DocumentPage and DocumentPageTransformation circular dependecy --- mayan/apps/documents/__init__.py | 1 - mayan/apps/documents/models.py | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mayan/apps/documents/__init__.py b/mayan/apps/documents/__init__.py index 730fbb688b..7a3c4c8616 100644 --- a/mayan/apps/documents/__init__.py +++ b/mayan/apps/documents/__init__.py @@ -154,7 +154,6 @@ namespace = StatisticNamespace(name='documents', label=_(u'Documents')) namespace.add_statistic(DocumentStatistics(name='document_stats', label=_(u'Document tendencies'))) namespace.add_statistic(DocumentUsageStatistics(name='document_usage', label=_(u'Document usage'))) - endpoint = APIEndPoint('documents') endpoint.register_urls(api_urls) endpoint.add_endpoint('document-list') diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 52fd1943a7..659bc83973 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -565,9 +565,6 @@ class DocumentPage(models.Model): verbose_name = _(u'document page') verbose_name_plural = _(u'document pages') - def get_transformation_list(self): - return DocumentPageTransformation.objects.get_for_document_page_as_list(self) - @models.permalink def get_absolute_url(self): return ('document_page_view', [self.pk]) @@ -641,3 +638,8 @@ class RecentDocument(models.Model): ordering = ('-datetime_accessed',) verbose_name = _(u'recent document') verbose_name_plural = _(u'recent documents') + + +# Quick hack to break the DocumentPage and DocumentPageTransformation circular dependency +# Can be remove once the transformations are moved to the converter app +DocumentPage.add_to_class('get_transformation_list', lambda document_page: DocumentPageTransformation.objects.get_for_document_page_as_list(document_page)) From 29eca11c50ae51cab499734e27b219a802374d74 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:42:18 -0400 Subject: [PATCH 24/29] Enable API pagination --- mayan/settings.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mayan/settings.py b/mayan/settings.py index ab42632178..eb4a3386c1 100644 --- a/mayan/settings.py +++ b/mayan/settings.py @@ -262,6 +262,12 @@ SERIALIZATION_MODULES = { SOUTH_MIGRATION_MODULES = { 'taggit': 'taggit.south_migrations', } +# ---------- Django REST framework ----------- +REST_FRAMEWORK = { + 'PAGINATE_BY': 10, + 'PAGINATE_BY_PARAM': 'page_size', + 'MAX_PAGINATE_BY': 100, +} try: from settings_local import * From a40fe3719b16664b207ca9b7f9a6e36618cda063 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:47:52 -0400 Subject: [PATCH 25/29] Rename the resources.py module to serializers.py --- mayan/apps/documents/api.py | 2 +- mayan/apps/documents/{resources.py => serializers.py} | 0 mayan/apps/sources/api.py | 11 +---------- mayan/apps/sources/registry.py | 7 ------- mayan/apps/sources/{resources.py => serializers.py} | 0 5 files changed, 2 insertions(+), 18 deletions(-) rename mayan/apps/documents/{resources.py => serializers.py} (100%) rename mayan/apps/sources/{resources.py => serializers.py} (100%) diff --git a/mayan/apps/documents/api.py b/mayan/apps/documents/api.py index c3726cf683..4db45e85fc 100644 --- a/mayan/apps/documents/api.py +++ b/mayan/apps/documents/api.py @@ -17,7 +17,7 @@ from permissions.models import Permission from .conf.settings import DISPLAY_SIZE, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL from .models import Document, DocumentVersion, DocumentPage from .permissions import PERMISSION_DOCUMENT_VIEW -from .resources import DocumentSerializer, DocumentVersionSerializer, DocumentPageSerializer +from .serializers import DocumentSerializer, DocumentVersionSerializer, DocumentPageSerializer logger = logging.getLogger(__name__) diff --git a/mayan/apps/documents/resources.py b/mayan/apps/documents/serializers.py similarity index 100% rename from mayan/apps/documents/resources.py rename to mayan/apps/documents/serializers.py diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py index 78818aba37..914e2f4b34 100644 --- a/mayan/apps/sources/api.py +++ b/mayan/apps/sources/api.py @@ -12,12 +12,9 @@ from converter.exceptions import UnkownConvertError, UnknownFileFormat from converter.literals import DEFAULT_PAGE_NUMBER, DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from documents.conf.settings import DISPLAY_SIZE, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL -#from acls.models import AccessEntry -#from permissions.models import Permission from .classes import StagingFile -from .resources import SerializerStagingFolder, SerializerStagingFolderFile from .models import StagingFolder -#from .permissions import PERMISSION_DOCUMENT_VIEW +from .serializers import SerializerStagingFolder, SerializerStagingFolderFile logger = logging.getLogger(__name__) @@ -57,12 +54,6 @@ class APIStagingSourceFileImageView(generics.GenericAPIView): staging_folder = get_object_or_404(StagingFolder, pk=staging_folder_pk) staging_file = staging_folder.get_file(encoded_filename=filename) - #try: - # Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_VIEW]) - #except PermissionDenied: - # AccessEntry.objects.check_access(PERMISSION_DOCUMENT_VIEW, request.user, document) - # Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION]) - size = request.GET.get('size', DISPLAY_SIZE) page = int(request.GET.get('page', DEFAULT_PAGE_NUMBER)) diff --git a/mayan/apps/sources/registry.py b/mayan/apps/sources/registry.py index 2e4b6bdd54..8056fb1dda 100644 --- a/mayan/apps/sources/registry.py +++ b/mayan/apps/sources/registry.py @@ -4,11 +4,4 @@ from django.conf.urls import url from .cleanup import cleanup - cleanup_functions = [cleanup] - -#version_0_api_services = [ -# #{'urlpattern': url(r'^staging_file/(?P[0-9]+)/image/$', APIStagingSourceView.as_view(), name='staging-file-image'), 'description': 'Return a base64 image of the staging file', 'url': 'staging_file//image/?page=&zoom=&rotate='}, -# {'urlpattern': url(r'^staging_folder/(?P[0-9]+)/$', APIStagingSourceView.as_view(), name='staging-source-list'), 'description': '', 'url': 'staging_file//image/?page=&zoom=&rotate='}, -#] - diff --git a/mayan/apps/sources/resources.py b/mayan/apps/sources/serializers.py similarity index 100% rename from mayan/apps/sources/resources.py rename to mayan/apps/sources/serializers.py From a4a443ab6d3cf08dda9a4680a43d91b2014f2056 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:50:29 -0400 Subject: [PATCH 26/29] Cleanups --- mayan/apps/sources/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mayan/apps/sources/api.py b/mayan/apps/sources/api.py index 914e2f4b34..89cf5f6067 100644 --- a/mayan/apps/sources/api.py +++ b/mayan/apps/sources/api.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import logging -from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404 from rest_framework import generics @@ -12,7 +11,6 @@ from converter.exceptions import UnkownConvertError, UnknownFileFormat from converter.literals import DEFAULT_PAGE_NUMBER, DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from documents.conf.settings import DISPLAY_SIZE, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL -from .classes import StagingFile from .models import StagingFolder from .serializers import SerializerStagingFolder, SerializerStagingFolderFile @@ -20,6 +18,7 @@ logger = logging.getLogger(__name__) # API Views + class APIStagingSourceFileView(generics.GenericAPIView): """ Details of the selected staging file. From 1c80d3327ecf928d22a5b45e71d890964f1c1f81 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Jul 2014 23:55:04 -0400 Subject: [PATCH 27/29] Use Python's json library --- mayan/apps/common/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/common/views.py b/mayan/apps/common/views.py index 80c6c7947a..707bdd687e 100644 --- a/mayan/apps/common/views.py +++ b/mayan/apps/common/views.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +from json import dumps, loads + from django.contrib import messages from django.contrib.auth.models import User from django.contrib.auth.views import login, password_change @@ -10,7 +12,6 @@ from django.http import HttpResponseRedirect from django.shortcuts import redirect, render_to_response from django.template import RequestContext from django.utils.http import urlencode -from django.utils.simplejson import dumps, loads from django.utils.translation import ugettext_lazy as _ from django.views.generic.edit import CreateView, DeleteView, UpdateView from django.views.generic.list import ListView From 88de6cfec959b21902ea13b70a045f934805345c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Jul 2014 00:21:32 -0400 Subject: [PATCH 28/29] Simplify translation string (was causing misterious test failure) --- mayan/apps/documents/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 659bc83973..9e87965082 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -608,7 +608,7 @@ class DocumentPageTransformation(models.Model): document_page = models.ForeignKey(DocumentPage, verbose_name=_(u'document page')) order = models.PositiveIntegerField(default=0, blank=True, null=True, verbose_name=_(u'order'), db_index=True) transformation = models.CharField(choices=get_available_transformations_choices(), max_length=128, verbose_name=_(u'transformation')) - arguments = models.TextField(blank=True, null=True, verbose_name=_(u'arguments'), help_text=_(u'Use dictionaries to indentify arguments, example: %s') % u'{\'degrees\':90}', validators=[ArgumentsValidator()]) + arguments = models.TextField(blank=True, null=True, verbose_name=_(u'arguments'), help_text=_(u'Use dictionaries to indentify arguments, example: {\'degrees\':90}'), validators=[ArgumentsValidator()]) objects = DocumentPageTransformationManager() def __unicode__(self): From 08661401f7a69da56260de9fd10e1c35e5bbeba3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Jul 2014 00:51:12 -0400 Subject: [PATCH 29/29] Simplify transformation choices label (evaluation of ugettext_lazy during model import was causing strange unrelated test failures) --- mayan/apps/converter/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mayan/apps/converter/api.py b/mayan/apps/converter/api.py index 320210ff60..be0d1242d3 100644 --- a/mayan/apps/converter/api.py +++ b/mayan/apps/converter/api.py @@ -118,8 +118,7 @@ def get_page_count(input_filepath): def get_available_transformations_choices(): result = [] for transformation in backend.get_available_transformations(): - transformation_template = u'%s %s' % (TRANSFORMATION_CHOICES[transformation]['label'], u','.join(['<%s>' % argument['name'] if argument['required'] else '[%s]' % argument['name'] for argument in TRANSFORMATION_CHOICES[transformation]['arguments']])) - result.append([transformation, transformation_template]) + result.append((transformation, TRANSFORMATION_CHOICES[transformation]['label'])) return result