from __future__ import unicode_literals import base64 import hashlib import logging import os import uuid from django.contrib.auth.models import User from django.core.files import File from django.core.urlresolvers import reverse from django.db import models, transaction from django.utils.encoding import python_2_unicode_compatible from django.utils.timezone import now from django.utils.translation import ugettext, ugettext_lazy as _ from common.literals import TIME_DELTA_UNIT_CHOICES from common.settings import setting_temporary_directory from converter import ( converter_class, TransformationResize, TransformationRotate, TransformationZoom ) from converter.exceptions import ( InvalidOfficeFormat, PageCountError ) from converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION from converter.models import Transformation from mimetype.api import get_mimetype from .events import ( event_document_create, event_document_new_version, event_document_properties_edit, event_document_version_revert ) from .literals import DEFAULT_DELETE_PERIOD, DEFAULT_DELETE_TIME_UNIT from .managers import ( DocumentManager, DocumentTypeManager, PassthroughManager, RecentDocumentManager, TrashCanManager ) from .runtime import cache_storage_backend, storage_backend from .settings import ( setting_display_size, setting_language, setting_language_choices, setting_zoom_max_level, setting_zoom_min_level ) from .signals import ( post_document_created, post_document_type_change, post_version_upload ) # document image cache name hash function HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() logger = logging.getLogger(__name__) def UUID_FUNCTION(*args, **kwargs): return unicode(uuid.uuid4()) @python_2_unicode_compatible class DocumentType(models.Model): """ Define document types or classes to which a specific set of properties can be attached """ label = models.CharField( max_length=32, unique=True, verbose_name=_('Label') ) trash_time_period = models.PositiveIntegerField( blank=True, help_text=_( 'Amount of time after which documents of this type will be ' 'moved to the trash.' ), null=True, verbose_name=_('Trash time period') ) trash_time_unit = models.CharField( blank=True, choices=TIME_DELTA_UNIT_CHOICES, null=True, max_length=8, verbose_name=_('Trash time unit') ) delete_time_period = models.PositiveIntegerField( default=DEFAULT_DELETE_PERIOD, help_text=_( 'Amount of time after which documents of this type in the trash ' 'will be deleted.' ), verbose_name=_('Delete time period') ) delete_time_unit = models.CharField( choices=TIME_DELTA_UNIT_CHOICES, default=DEFAULT_DELETE_TIME_UNIT, max_length=8, verbose_name=_('Delete time unit') ) objects = DocumentTypeManager() class Meta: ordering = ('label',) verbose_name = _('Document type') verbose_name_plural = _('Documents types') def __str__(self): return self.label def delete(self, *args, **kwargs): for document in Document.passthrough.filter(document_type=self): document.delete(to_trash=False) return super(DocumentType, self).delete(*args, **kwargs) @property def deleted_documents(self): return DeletedDocument.objects.filter(document_type=self) def new_document(self, file_object, label=None, description=None, language=None, _user=None): try: with transaction.atomic(): document = self.documents.create( description=description or '', label=label or unicode(file_object), language=language or setting_language.value ) document.save(_user=_user) document.new_version(file_object=file_object, _user=_user) return document except Exception as exception: logger.critical( 'Unexpected exception while trying to create new document ' '"%s" from document type "%s"; %s', label or unicode(file_object), self, exception ) raise @python_2_unicode_compatible class Document(models.Model): """ Defines a single document with it's fields and properties """ uuid = models.CharField( default=UUID_FUNCTION, editable=False, max_length=48 ) document_type = models.ForeignKey( DocumentType, related_name='documents', verbose_name=_('Document type') ) label = models.CharField( blank=True, db_index=True, default='', max_length=255, help_text=_('The name of the document'), verbose_name=_('Label') ) description = models.TextField( blank=True, default='', verbose_name=_('Description') ) date_added = models.DateTimeField( auto_now_add=True, db_index=True, verbose_name=_('Added') ) language = models.CharField( blank=True, choices=setting_language_choices.value, default=setting_language.value, max_length=8, verbose_name=_('Language') ) in_trash = models.BooleanField( default=False, editable=False, verbose_name=_('In trash?') ) # TODO: set editable to False deleted_date_time = models.DateTimeField( blank=True, editable=True, null=True, verbose_name=_('Date and time trashed') ) is_stub = models.BooleanField( default=True, editable=False, help_text=_( 'A document stub is a document with an entry on the database but ' 'no file uploaded. This could be an interrupted upload or a ' 'deferred upload via the API.'), verbose_name=_('Is stub?') ) objects = DocumentManager() passthrough = PassthroughManager() trash = TrashCanManager() class Meta: verbose_name = _('Document') verbose_name_plural = _('Documents') ordering = ('-date_added',) def __str__(self): return self.label or ugettext('Document stub, id: %d') % self.pk def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) if not self.in_trash and to_trash: self.in_trash = True self.deleted_date_time = now() self.save() else: for version in self.versions.all(): version.delete() return super(Document, self).delete(*args, **kwargs) def get_absolute_url(self): return reverse('documents:document_preview', args=(self.pk,)) def save(self, *args, **kwargs): user = kwargs.pop('_user', None) new_document = not self.pk super(Document, self).save(*args, **kwargs) if new_document: if user: self.add_as_recent_document_for_user(user) event_document_create.commit(actor=user, target=self) else: event_document_create.commit(target=self) else: event_document_properties_edit.commit(actor=user, target=self) # Custom methods def add_as_recent_document_for_user(self, user): RecentDocument.objects.add_document_for_user(user, self) def exists(self): """ Returns a boolean value that indicates if the document's latest version file exists in storage """ return self.latest_version.exists() def invalidate_cache(self): for document_version in self.versions.all(): document_version.invalidate_cache() def new_version(self, file_object, comment=None, _user=None): logger.info('Creating new document version for document: %s', self) document_version = DocumentVersion( document=self, comment=comment or '', file=File(file_object) ) document_version.save(_user=_user) logger.info('New document version queued for document: %s', self) return document_version def open(self, *args, **kwargs): """ Return a file descriptor to a document's file irrespective of the storage backend """ return self.latest_version.open(*args, **kwargs) def restore(self): self.in_trash = False self.save() def save_to_file(self, *args, **kwargs): return self.latest_version.save_to_file(*args, **kwargs) def set_document_type(self, document_type, force=False): has_changed = self.document_type != document_type self.document_type = document_type self.save() if has_changed or force: post_document_type_change.send( sender=self.__class__, instance=self ) @property def size(self): return self.latest_version.size # Compatibility methods @property def checksum(self): return self.latest_version.checksum @property def date_updated(self): return self.latest_version.timestamp # TODO: rename to file_encoding @property def file_mime_encoding(self): return self.latest_version.encoding @property def file_mimetype(self): return self.latest_version.mimetype @property def latest_version(self): return self.versions.order_by('timestamp').last() @property def page_count(self): return self.latest_version.page_count @property def pages(self): try: return self.latest_version.pages except AttributeError: # Document has no version yet return 0 @property def signature_state(self): return self.latest_version.signature_state class DeletedDocument(Document): objects = TrashCanManager() class Meta: proxy = True @python_2_unicode_compatible class DocumentVersion(models.Model): """ Model that describes a document version and its properties """ _pre_open_hooks = {} _post_save_hooks = {} @classmethod def register_pre_open_hook(cls, order, func): cls._pre_open_hooks[order] = func @classmethod def register_post_save_hook(cls, order, func): cls._post_save_hooks[order] = func document = models.ForeignKey( Document, related_name='versions', verbose_name=_('Document') ) timestamp = models.DateTimeField( auto_now_add=True, db_index=True, verbose_name=_('Timestamp') ) comment = models.TextField( blank=True, default='', verbose_name=_('Comment') ) # File related fields file = models.FileField( storage=storage_backend, upload_to=UUID_FUNCTION, verbose_name=_('File') ) mimetype = models.CharField( blank=True, editable=False, max_length=255, null=True ) encoding = models.CharField( blank=True, editable=False, max_length=64, null=True ) checksum = models.TextField( blank=True, editable=False, null=True, verbose_name=_('Checksum') ) class Meta: verbose_name = _('Document version') verbose_name_plural = _('Document version') def __str__(self): return '{0} - {1}'.format(self.document, self.timestamp) def delete(self, *args, **kwargs): for page in self.pages.all(): page.delete() self.file.storage.delete(self.file.name) return super(DocumentVersion, self).delete(*args, **kwargs) def save(self, *args, **kwargs): """ Overloaded save method that updates the document version's checksum, mimetype, and page count when created """ user = kwargs.pop('_user', None) new_document_version = not self.pk if new_document_version: logger.info('Creating new version for document: %s', self.document) try: with transaction.atomic(): super(DocumentVersion, self).save(*args, **kwargs) for key in sorted(DocumentVersion._post_save_hooks): DocumentVersion._post_save_hooks[key](self) if new_document_version: # Only do this for new documents self.update_checksum(save=False) self.update_mimetype(save=False) self.save() self.update_page_count(save=False) logger.info( 'New document version "%s" created for document: %s', self, self.document ) self.document.is_stub = False if not self.document.label: self.document.label = unicode(self.file) self.document.save() except Exception as exception: logger.error( 'Error creating new document version for document "%s"; %s', self.document, exception ) raise else: if new_document_version: event_document_new_version.commit( actor=user, target=self.document ) post_version_upload.send(sender=self.__class__, instance=self) if tuple(self.document.versions.all()) == (self,): post_document_created.send( sender=self.document.__class__, instance=self.document ) # Custom methods @property def cache_filename(self): return 'document-version-{}'.format(self.uuid) def exists(self): """ Returns a boolean value that indicates if the document's file exists in storage """ return self.file.storage.exists(self.file.name) def get_intermidiate_file(self): cache_filename = self.cache_filename logger.debug('Intermidiate filename: %s', cache_filename) if cache_storage_backend.exists(cache_filename): logger.debug('Intermidiate file "%s" found.', cache_filename) return cache_storage_backend.open(cache_filename) else: logger.debug('Intermidiate file "%s" not found.', cache_filename) try: converter = converter_class(file_object=self.open()) pdf_file_object = converter.to_pdf() with cache_storage_backend.open(cache_filename, 'wb+') as file_object: for chunk in pdf_file_object: file_object.write(chunk) return cache_storage_backend.open(cache_filename) except InvalidOfficeFormat: return self.open() except Exception as exception: # Cleanup in case of error logger.error( 'Error creating intermediate file "%s"; %s.', cache_filename, exception ) cache_storage_backend.delete(cache_filename) raise def invalidate_cache(self): cache_storage_backend.delete(self.cache_filename) for page in self.pages.all(): page.invalidate_cache() def open(self, raw=False): """ Return a file descriptor to a document version's file irrespective of the storage backend """ if raw: return self.file.storage.open(self.file.name) else: result = self.file.storage.open(self.file.name) for key in sorted(DocumentVersion._pre_open_hooks): result = DocumentVersion._pre_open_hooks[key](result, self) return result @property def page_count(self): return self.pages.count() def revert(self, _user=None): """ Delete the subsequent versions after this one """ logger.info( 'Reverting to document document: %s to version: %s', self.document, self ) event_document_version_revert.commit(actor=_user, target=self.document) for version in self.document.versions.filter(timestamp__gt=self.timestamp): version.delete() def save_to_file(self, filepath, buffer_size=1024 * 1024): """ Save a copy of the document from the document storage backend to the local filesystem """ input_descriptor = self.open() output_descriptor = open(filepath, 'wb') while True: copy_buffer = input_descriptor.read(buffer_size) if copy_buffer: output_descriptor.write(copy_buffer) else: break output_descriptor.close() input_descriptor.close() return filepath @property def size(self): if self.exists(): return self.file.storage.size(self.file.name) else: return None def update_checksum(self, save=True): """ Open a document version's file and update the checksum field using the user provided checksum function """ if self.exists(): source = self.open() self.checksum = unicode(HASH_FUNCTION(source.read())) source.close() if save: self.save() def update_mimetype(self, save=True): """ Read a document verions's file and determine the mimetype by calling the get_mimetype wrapper """ if self.exists(): try: with self.open() as file_object: self.mimetype, self.encoding = get_mimetype( file_object=file_object ) except: self.mimetype = '' self.encoding = '' finally: if save: self.save() def update_page_count(self, save=True): try: with self.open() as file_object: converter = converter_class( file_object=file_object, mime_type=self.mimetype ) detected_pages = converter.get_page_count() except PageCountError: # If converter backend doesn't understand the format, # use 1 as the total page count pass else: with transaction.atomic(): self.pages.all().delete() for page_number in range(detected_pages): DocumentPage.objects.create( document_version=self, page_number=page_number + 1 ) # TODO: is this needed anymore if save: self.save() return detected_pages @property def uuid(self): # Make cache UUID a mix of document UUID, version ID return '{}-{}'.format(self.document.uuid, self.pk) @python_2_unicode_compatible class DocumentTypeFilename(models.Model): """ List of labels available to a specific document type for the quick rename functionality """ document_type = models.ForeignKey( DocumentType, related_name='filenames', verbose_name=_('Document type') ) filename = models.CharField( db_index=True, max_length=128, verbose_name=_('Label') ) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) class Meta: ordering = ('filename',) unique_together = ('document_type', 'filename') verbose_name = _('Quick rename template') verbose_name_plural = _('Quick rename templates') def __str__(self): return self.filename @python_2_unicode_compatible class DocumentPage(models.Model): """ Model that describes a document version page """ document_version = models.ForeignKey( DocumentVersion, related_name='pages', verbose_name=_('Document version') ) page_number = models.PositiveIntegerField( db_index=True, default=1, editable=False, verbose_name=_('Page number') ) class Meta: ordering = ('page_number',) verbose_name = _('Document page') verbose_name_plural = _('Document pages') def __str__(self): return _( 'Page %(page_num)d out of %(total_pages)d of %(document)s' ) % { 'document': unicode(self.document), 'page_num': self.page_number, 'total_pages': self.document_version.pages.count() } def delete(self, *args, **kwargs): self.invalidate_cache() super(DocumentPage, self).delete(*args, **kwargs) def get_absolute_url(self): return reverse('documents:document_page_view', args=(self.pk,)) # Custom methods @property def cache_filename(self): return 'page-cache-{}'.format(self.uuid) @property def document(self): return self.document_version.document def get_image(self, *args, **kwargs): as_base64 = kwargs.pop('as_base64', False) transformations = kwargs.pop('transformations', []) size = kwargs.pop('size', setting_display_size.value) rotation = int( kwargs.pop('rotation', DEFAULT_ROTATION) or DEFAULT_ROTATION ) zoom_level = int( kwargs.pop('zoom', DEFAULT_ZOOM_LEVEL) or DEFAULT_ZOOM_LEVEL ) if zoom_level < setting_zoom_min_level.value: zoom_level = setting_zoom_min_level.value if zoom_level > setting_zoom_max_level.value: zoom_level = setting_zoom_max_level.value rotation = rotation % 360 cache_filename = self.cache_filename logger.debug('Page cache filename: %s', cache_filename) if cache_storage_backend.exists(cache_filename): logger.debug('Page cache file "%s" found', cache_filename) converter = converter_class( file_object=cache_storage_backend.open(cache_filename) ) converter.seek(0) else: logger.debug('Page cache file "%s" not found', cache_filename) try: converter = converter_class( file_object=self.document_version.get_intermidiate_file() ) converter.seek(page_number=self.page_number - 1) page_image = converter.get_page() with cache_storage_backend.open(cache_filename, 'wb+') as file_object: file_object.write(page_image.getvalue()) except Exception as exception: # Cleanup in case of error logger.error( 'Error creating page cache file "%s"; %s', cache_filename, exception ) cache_storage_backend.delete(cache_filename) raise # Stored transformations for stored_transformation in Transformation.objects.get_for_model(self, as_classes=True): converter.transform(transformation=stored_transformation) # Interactive transformations for transformation in transformations: converter.transform(transformation=transformation) if rotation: converter.transform(transformation=TransformationRotate( degrees=rotation) ) if size: converter.transform(transformation=TransformationResize( **dict(zip(('width', 'height'), (size.split('x'))))) ) if zoom_level: converter.transform( transformation=TransformationZoom(percent=zoom_level) ) page_image = converter.get_page() if as_base64: # TODO: don't prepend 'data:%s;base64,%s' part return 'data:%s;base64,%s' % ( 'image/png', base64.b64encode(page_image.getvalue()) ) else: return page_image def invalidate_cache(self): cache_storage_backend.delete(self.cache_filename) @property def siblings(self): return DocumentPage.objects.filter( document_version=self.document_version ) @property def uuid(self): """ Make cache UUID a mix of version ID and page ID to avoid using stale images """ return '{}-{}'.format(self.document_version.uuid, self.pk) @python_2_unicode_compatible class RecentDocument(models.Model): """ Keeps a list of the n most recent accessed or created document for a given user """ user = models.ForeignKey( User, db_index=True, editable=False, verbose_name=_('User') ) document = models.ForeignKey( Document, editable=False, verbose_name=_('Document') ) datetime_accessed = models.DateTimeField( auto_now=True, db_index=True, verbose_name=_('Accessed') ) objects = RecentDocumentManager() class Meta: ordering = ('-datetime_accessed',) verbose_name = _('Recent document') verbose_name_plural = _('Recent documents') def __str__(self): return unicode(self.document)