from __future__ import unicode_literals import base64 import hashlib import logging import os import uuid from django.contrib.auth.models import User from django.core.urlresolvers import reverse from django.db import models, transaction from django.utils.encoding import python_2_unicode_compatible from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ from common.literals import TIME_DELTA_UNIT_CHOICES from common.models import SharedUploadedFile from common.settings import setting_temporary_directory from common.utils import fs_cleanup from converter import ( converter_class, TransformationResize, TransformationRotate, TransformationZoom ) from converter.exceptions import InvalidOfficeFormat, UnknownFileFormat from converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION from converter.models import Transformation from mimetype.api import get_mimetype from .events import ( event_document_create, event_document_new_version, event_document_version_revert ) from .literals import DEFAULT_DELETE_PERIOD, DEFAULT_DELETE_TIME_UNIT from .managers import ( DocumentManager, DocumentTypeManager, PassthroughManager, RecentDocumentManager, TrashCanManager ) from .runtime import storage_backend from .settings import ( setting_cache_path, setting_display_size, setting_language, setting_language_choices, setting_zoom_max_level, setting_zoom_min_level ) from .signals import ( post_document_created, post_document_type_change, post_version_upload ) HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() # document image cache name hash function logger = logging.getLogger(__name__) def UUID_FUNCTION(*args, **kwargs): return unicode(uuid.uuid4()) @python_2_unicode_compatible class DocumentType(models.Model): """ Define document types or classes to which a specific set of properties can be attached """ label = models.CharField(max_length=32, unique=True, verbose_name=_('Label')) trash_time_period = models.PositiveIntegerField(blank=True, help_text=_('Amount of time after which documents of this type will be moved to the trash.'), null=True, verbose_name=_('Trash time period')) trash_time_unit = models.CharField(blank=True, choices=TIME_DELTA_UNIT_CHOICES, null=True, max_length=8, verbose_name=_('Trash time unit')) delete_time_period = models.PositiveIntegerField(default=DEFAULT_DELETE_PERIOD, help_text=_('Amount of time after which documents of this type in the trash will be deleted.'), verbose_name=_('Delete time period')) delete_time_unit = models.CharField(choices=TIME_DELTA_UNIT_CHOICES, default=DEFAULT_DELETE_TIME_UNIT, max_length=8, verbose_name=_('Delete time unit')) objects = DocumentTypeManager() def __str__(self): return self.label def delete(self, *args, **kwargs): for document in Document.passthrough.filter(document_type=self): document.delete(to_trash=False) return super(DocumentType, self).delete(*args, **kwargs) def new_document(self, file_object, label=None, description=None, language=None, _user=None): if not language: language = setting_language.value if not label: label = unicode(file_object) document = self.documents.create(description=description or '', language=language, label=label) document.save(_user=_user) document.new_version(file_object=file_object, _user=_user) return document class Meta: verbose_name = _('Document type') verbose_name_plural = _('Documents types') ordering = ('label',) @python_2_unicode_compatible class Document(models.Model): """ Defines a single document with it's fields and properties """ uuid = models.CharField(default=UUID_FUNCTION, editable=False, max_length=48) document_type = models.ForeignKey(DocumentType, related_name='documents', verbose_name=_('Document type')) label = models.CharField(db_index=True, default=_('Uninitialized document'), max_length=255, help_text=_('The name of the document'), verbose_name=_('Label')) description = models.TextField(blank=True, verbose_name=_('Description')) date_added = models.DateTimeField(auto_now_add=True, verbose_name=_('Added')) language = models.CharField(choices=setting_language_choices.value, default=setting_language.value, max_length=8, verbose_name=_('Language')) in_trash = models.BooleanField(default=False, editable=False, verbose_name=_('In trash?')) deleted_date_time = models.DateTimeField(blank=True, editable=True, null=True, verbose_name=_('Date and time trashed')) is_stub = models.BooleanField(default=True, editable=False, verbose_name=_('Is stub?')) objects = DocumentManager() passthrough = PassthroughManager() trash = TrashCanManager() class Meta: verbose_name = _('Document') verbose_name_plural = _('Documents') ordering = ('-date_added',) def set_document_type(self, document_type, force=False): has_changed = self.document_type != document_type self.document_type = document_type self.save() if has_changed or force: post_document_type_change.send(sender=self.__class__, instance=self) def invalidate_cache(self): for document_version in self.versions.all(): document_version.invalidate_cache() def __str__(self): return self.label def get_absolute_url(self): return reverse('documents:document_preview', args=[self.pk]) def save(self, *args, **kwargs): user = kwargs.pop('_user', None) new_document = not self.pk super(Document, self).save(*args, **kwargs) if new_document: if user: self.add_as_recent_document_for_user(user) event_document_create.commit(actor=user, target=self) else: event_document_create.commit(target=self) def add_as_recent_document_for_user(self, user): RecentDocument.objects.add_document_for_user(user, self) def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) if not self.in_trash and to_trash: self.in_trash = True self.deleted_date_time = now() self.save() else: for version in self.versions.all(): version.delete() return super(Document, self).delete(*args, **kwargs) def restore(self): self.in_trash = False self.save() @property def size(self): return self.latest_version.size def new_version(self, file_object, comment=None, _user=None): from .tasks import task_upload_new_version logger.info('Queueing creation of a new document version for document: %s', self) shared_uploaded_file = SharedUploadedFile.objects.create(file=file_object) if _user: user_id = _user.pk else: user_id = None task_upload_new_version.apply_async(kwargs=dict( shared_uploaded_file_id=shared_uploaded_file.pk, document_id=self.pk, user_id=user_id, ), queue='uploads') logger.info('New document version created for document: %s', self) # Proxy methods def open(self, *args, **kwargs): """ Return a file descriptor to a document's file irrespective of the storage backend """ return self.latest_version.open(*args, **kwargs) def save_to_file(self, *args, **kwargs): return self.latest_version.save_to_file(*args, **kwargs) def exists(self): """ Returns a boolean value that indicates if the document's latest version file exists in storage """ return self.latest_version.exists() # Compatibility methods @property def file(self): return self.latest_version.file @property def file_mimetype(self): return self.latest_version.mimetype # TODO: rename to file_encoding @property def file_mime_encoding(self): return self.latest_version.encoding @property def date_updated(self): return self.latest_version.timestamp @property def checksum(self): return self.latest_version.checksum @property def signature_state(self): return self.latest_version.signature_state @property def pages(self): try: return self.latest_version.pages except AttributeError: # Document has no version yet return 0 @property def page_count(self): return self.latest_version.page_count @property def latest_version(self): return self.versions.order_by('timestamp').last() def document_save_to_temp_dir(self, filename, buffer_size=1024 * 1024): temporary_path = os.path.join(setting_temporary_directory.value, filename) return self.save_to_file(temporary_path, buffer_size) class DeletedDocument(Document): objects = TrashCanManager() class Meta: proxy = True @python_2_unicode_compatible class DocumentVersion(models.Model): """ Model that describes a document version and its properties """ _pre_open_hooks = {} _post_save_hooks = {} @classmethod def register_pre_open_hook(cls, order, func): cls._pre_open_hooks[order] = func @classmethod def register_post_save_hook(cls, order, func): cls._post_save_hooks[order] = func document = models.ForeignKey(Document, related_name='versions', verbose_name=_('Document')) timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_('Timestamp')) comment = models.TextField(blank=True, verbose_name=_('Comment')) # File related fields file = models.FileField(storage=storage_backend, upload_to=UUID_FUNCTION, verbose_name=_('File')) mimetype = models.CharField(blank=True, editable=False, max_length=255, null=True) encoding = models.CharField(blank=True, editable=False, max_length=64, null=True) checksum = models.TextField(blank=True, editable=False, null=True, verbose_name=_('Checksum')) class Meta: verbose_name = _('Document version') verbose_name_plural = _('Document version') def __str__(self): return '{0} - {1}'.format(self.document, self.timestamp) def save(self, *args, **kwargs): """ Overloaded save method that updates the document version's checksum, mimetype, and page count when created """ user = kwargs.pop('_user', None) new_document_version = not self.pk if new_document_version: logger.info('Creating new version for document: %s', self.document) try: with transaction.atomic(): super(DocumentVersion, self).save(*args, **kwargs) for key in sorted(DocumentVersion._post_save_hooks): DocumentVersion._post_save_hooks[key](self) if new_document_version: # Only do this for new documents self.update_checksum(save=False) self.update_mimetype(save=False) self.save() self.update_page_count(save=False) logger.info('New document version "%s" created for document: %s', self, self.document) self.document.is_stub = False self.document.save() except Exception as exception: logger.error('Error creating new document version for document "%s"; %s', self.document, exception) raise else: if new_document_version: event_document_new_version.commit(actor=user, target=self.document) post_version_upload.send(sender=self.__class__, instance=self) if tuple(self.document.versions.all()) == (self,): post_document_created.send(sender=self.document.__class__, instance=self.document) def invalidate_cache(self): for page in self.pages.all(): page.invalidate_cache() def update_checksum(self, save=True): """ Open a document version's file and update the checksum field using the user provided checksum function """ if self.exists(): source = self.open() self.checksum = unicode(HASH_FUNCTION(source.read())) source.close() if save: self.save() def update_page_count(self, save=True): try: with self.open() as file_object: converter = converter_class(file_object=file_object, mime_type=self.mimetype) detected_pages = converter.get_page_count() except UnknownFileFormat: # If converter backend doesn't understand the format, # use 1 as the total page count detected_pages = 1 # TODO: should be no pages instead? with transaction.atomic(): self.pages.all().delete() for page_number in range(detected_pages): DocumentPage.objects.create( document_version=self, page_number=page_number + 1 ) # TODO: is this needed anymore if save: self.save() return detected_pages def revert(self, user=None): """ Delete the subsequent versions after this one """ logger.info('Reverting to document document: %s to version: %s', self.document, self) event_document_version_revert.commit(actor=user, target=self.document) for version in self.document.versions.filter(timestamp__gt=self.timestamp): version.delete() def update_mimetype(self, save=True): """ Read a document verions's file and determine the mimetype by calling the get_mimetype wrapper """ if self.exists(): try: with self.open() as file_object: self.mimetype, self.encoding = get_mimetype(file_object=file_object) except: self.mimetype = '' self.encoding = '' finally: if save: self.save() def delete(self, *args, **kwargs): for page in self.pages.all(): page.delete() self.file.storage.delete(self.file.path) return super(DocumentVersion, self).delete(*args, **kwargs) def exists(self): """ Returns a boolean value that indicates if the document's file exists in storage """ return self.file.storage.exists(self.file.path) def open(self, raw=False): """ Return a file descriptor to a document version's file irrespective of the storage backend """ if raw: return self.file.storage.open(self.file.path) else: result = self.file.storage.open(self.file.path) for key in sorted(DocumentVersion._pre_open_hooks): result = DocumentVersion._pre_open_hooks[key](result, self) return result def save_to_file(self, filepath, buffer_size=1024 * 1024): """ Save a copy of the document from the document storage backend to the local filesystem """ input_descriptor = self.open() output_descriptor = open(filepath, 'wb') while True: copy_buffer = input_descriptor.read(buffer_size) if copy_buffer: output_descriptor.write(copy_buffer) else: break output_descriptor.close() input_descriptor.close() return filepath @property def size(self): if self.exists(): return self.file.storage.size(self.file.path) else: return None @property def page_count(self): return self.pages.count() @property def uuid(self): # Make cache UUID a mix of document UUID, version ID return '{}-{}'.format(self.document.uuid, self.pk) @property def cache_filename(self): return os.path.join(setting_cache_path.value, 'document-version-{}'.format(self.uuid)) def get_intermidiate_file(self): cache_filename = self.cache_filename logger.debug('Intermidiate filename: %s', cache_filename) if os.path.exists(cache_filename): logger.debug('Intermidiate file "%s" found.', cache_filename) return open(cache_filename) else: logger.debug('Intermidiate file "%s" not found.', cache_filename) try: converter = converter_class(file_object=self.open()) pdf_file_object = converter.to_pdf() with open(cache_filename, 'wb+') as file_object: for chunk in pdf_file_object: file_object.write(chunk) return open(cache_filename) except InvalidOfficeFormat: return self.open() except Exception as exception: # Cleanup in case of error logger.error('Error creating intermediate file "%s"; %s.', cache_filename, exception) fs_cleanup(cache_filename) raise @python_2_unicode_compatible class DocumentTypeFilename(models.Model): """ List of filenames available to a specific document type for the quick rename functionality """ document_type = models.ForeignKey(DocumentType, related_name='filenames', verbose_name=_('Document type')) filename = models.CharField(db_index=True, max_length=128, verbose_name=_('Filename')) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) def __str__(self): return self.filename class Meta: ordering = ('filename',) unique_together = ('document_type', 'filename') verbose_name = _('Document type quick rename filename') verbose_name_plural = _('Document types quick rename filenames') @python_2_unicode_compatible class DocumentPage(models.Model): """ Model that describes a document version page """ document_version = models.ForeignKey(DocumentVersion, related_name='pages', verbose_name=_('Document version')) page_number = models.PositiveIntegerField(db_index=True, default=1, editable=False, verbose_name=_('Page number')) def __str__(self): return _('Page %(page_num)d out of %(total_pages)d of %(document)s') % { 'document': unicode(self.document), 'page_num': self.page_number, 'total_pages': self.document_version.pages.count() } class Meta: ordering = ('page_number',) verbose_name = _('Document page') verbose_name_plural = _('Document pages') def get_absolute_url(self): return reverse('documents:document_page_view', args=[self.pk]) def delete(self, *args, **kwargs): self.invalidate_cache() super(DocumentPage, self).delete(*args, **kwargs) @property def siblings(self): return DocumentPage.objects.filter(document_version=self.document_version) # Compatibility methods @property def document(self): return self.document_version.document def invalidate_cache(self): fs_cleanup(self.cache_filename) @property def uuid(self): """ Make cache UUID a mix of version ID and page ID to avoid using stale images """ return '{}-{}'.format(self.document_version.uuid, self.pk) @property def cache_filename(self): return os.path.join(setting_cache_path.value, 'page-cache-{}'.format(self.uuid)) def get_image(self, *args, **kwargs): as_base64 = kwargs.pop('as_base64', False) transformations = kwargs.pop('transformations', []) size = kwargs.pop('size', setting_display_size.value) rotation = kwargs.pop('rotation', DEFAULT_ROTATION) zoom_level = kwargs.pop('zoom', DEFAULT_ZOOM_LEVEL) if zoom_level < setting_zoom_min_level.value: zoom_level = setting_zoom_min_level.value if zoom_level > setting_zoom_max_level.value: zoom_level = setting_zoom_max_level.value rotation = rotation % 360 cache_filename = self.cache_filename logger.debug('Page cache filename: %s', cache_filename) if os.path.exists(cache_filename): logger.debug('Page cache file "%s" found', cache_filename) converter = converter_class(file_object=open(cache_filename)) converter.seek(0) else: logger.debug('Page cache file "%s" not found', cache_filename) try: converter = converter_class(file_object=self.document_version.get_intermidiate_file()) converter.seek(page_number=self.page_number - 1) page_image = converter.get_page() with open(cache_filename, 'wb+') as file_object: file_object.write(page_image.getvalue()) except Exception as exception: # Cleanup in case of error logger.error('Error creating page cache file "%s"; %s', cache_filename, exception) fs_cleanup(cache_filename) raise # Stored transformations for stored_transformation in Transformation.objects.get_for_model(self, as_classes=True): converter.transform(transformation=stored_transformation) # Interactive transformations for transformation in transformations: converter.transform(transformation=transformation) if rotation: converter.transform(transformation=TransformationRotate(degrees=rotation)) if size: converter.transform(transformation=TransformationResize(**dict(zip(('width', 'height'), (size.split('x')))))) if zoom_level: converter.transform(transformation=TransformationZoom(percent=zoom_level)) page_image = converter.get_page() if as_base64: # TODO: don't prepend 'data:%s;base64,%s' part return 'data:%s;base64,%s' % ('image/png', base64.b64encode(page_image.getvalue())) else: return page_image @python_2_unicode_compatible class RecentDocument(models.Model): """ Keeps a list of the n most recent accessed or created document for a given user """ user = models.ForeignKey(User, editable=False, verbose_name=_('User')) document = models.ForeignKey(Document, editable=False, verbose_name=_('Document')) datetime_accessed = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_('Accessed')) objects = RecentDocumentManager() def __str__(self): return unicode(self.document) class Meta: ordering = ('-datetime_accessed',) verbose_name = _('Recent document') verbose_name_plural = _('Recent documents')