Add support for quotas. GitLab issue #284.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
This commit is contained in:
Roberto Rosario
2017-08-02 04:10:55 -04:00
parent 20e3634f5a
commit 01420c42dd
18 changed files with 812 additions and 0 deletions

View File

@@ -4,6 +4,7 @@
- Content windows appearance changes
- Add new document's version list view permission
- Add support for notifications. GitLab #262.
- Add quota support.
2.6.4 (2017-07-26)
==================

View File

@@ -0,0 +1,3 @@
from __future__ import unicode_literals
default_app_config = 'quotas.apps.QuotasApp'

View File

@@ -0,0 +1,24 @@
from __future__ import unicode_literals
from django.contrib import admin
from .models import Quota
@admin.register(Quota)
class QuotaAdmin(admin.ModelAdmin):
list_display = (
'backend_path', 'backend_data', 'enabled', 'editable',
)
def has_change_permission(self, request, obj=None):
if obj:
return obj.editable
else:
return True
def has_delete_permission(self, request, obj=None):
if obj:
return obj.editable
else:
return False

85
mayan/apps/quotas/apps.py Normal file
View File

@@ -0,0 +1,85 @@
from __future__ import unicode_literals
from django.db.utils import OperationalError
from django.utils.translation import ugettext_lazy as _
from acls import ModelPermission
from acls.links import link_acl_list
from acls.permissions import permission_acl_edit, permission_acl_view
from common import MayanAppConfig, menu_object, menu_secondary, menu_setup
from common.widgets import two_state_template
from navigation import SourceColumn
from .classes import QuotaBackend
from .links import (
link_quota_create, link_quota_delete, link_quota_edit, link_quota_list,
link_quota_setup
)
from .permissions import (
permission_quota_delete, permission_quota_edit, permission_quota_view
)
class QuotasApp(MayanAppConfig):
name = 'quotas'
verbose_name = _('Quotas')
def ready(self, *args, **kwargs):
super(QuotasApp, self).ready(*args, **kwargs)
Quota = self.get_model('Quota')
QuotaBackend.initialize()
try:
for quota in Quota.objects.all():
quota.update_receiver()
except OperationalError:
# Ignore errors during migration
pass
ModelPermission.register(
model=Quota, permissions=(
permission_acl_edit, permission_acl_view,
permission_quota_delete, permission_quota_edit,
permission_quota_view
)
)
SourceColumn(
source=Quota, label=_('Backend'), attribute='backend_label'
)
SourceColumn(
source=Quota, label=_('Display'), attribute='backend_display'
)
SourceColumn(
source=Quota, label=_('Usage'), attribute='backend_usage'
)
SourceColumn(
source=Quota, label=_('Enabled?'),
func=lambda context: two_state_template(
context['object'].enabled
)
)
SourceColumn(
source=Quota, label=_('Editable?'),
func=lambda context: two_state_template(
context['object'].editable
)
)
menu_object.bind_links(
links=(
link_quota_edit, link_acl_list, link_quota_delete,
), sources=(Quota,)
)
menu_secondary.bind_links(
links=(
link_quota_list, link_quota_create,
), sources=(
Quota, 'quotas:quota_backend_selection', 'quotas:quota_create',
'quotas:quota_list',
)
)
menu_setup.bind_links(links=(link_quota_setup,))

View File

@@ -0,0 +1,81 @@
from __future__ import unicode_literals
from importlib import import_module
import logging
from django.apps import apps
from django.utils import six
from django.utils.encoding import force_text
logger = logging.getLogger(__name__)
__ALL__ = ('QuotaBackend',)
class QuotaBackendMetaclass(type):
_registry = {}
def __new__(mcs, name, bases, attrs):
new_class = super(QuotaBackendMetaclass, mcs).__new__(
mcs, name, bases, attrs
)
if not new_class.__module__ == 'quotas.classes':
mcs._registry[
'{}.{}'.format(new_class.__module__, name)
] = new_class
new_class.id = '{}.{}'.format(new_class.__module__, name)
return new_class
class QuotaBackendBase(object):
"""
Base class for the mailing backends. This class is mainly a wrapper
for other Django backends that adds a few metadata to specify the
fields it needs to be instanciated at runtime.
The fields attribute is a list of dictionaries with the format:
{
'name': '' # Field internal name
'label': '' # Label to show to users
'class': '' # Field class to use. Field classes are Python dot
paths to Django's form fields.
'initial': '' # Field initial value
'default': '' # Default value.
}
"""
fields = ()
class QuotaBackend(six.with_metaclass(QuotaBackendMetaclass, QuotaBackendBase)):
@classmethod
def get(cls, name):
return cls._registry[name]
@classmethod
def get_all(cls):
return sorted(
cls._registry.values(), key=lambda x: x.label
)
@classmethod
def as_choices(cls):
return [
(
backend.id, backend.label
) for backend in QuotaBackend.get_all()
]
@staticmethod
def initialize():
for app in apps.get_app_configs():
try:
import_module('{}.quota_backends'.format(app.name))
except ImportError as exception:
if force_text(exception) != 'No module named quota_backends':
logger.error(
'Error importing %s quota_backends.py file; %s',
app.name, exception
)

View File

@@ -0,0 +1,12 @@
from __future__ import unicode_literals
class QuotaBaseException(Exception):
"""
Base exception for the quota app
"""
pass
class QuotaExceeded(QuotaBaseException):
pass

View File

@@ -0,0 +1,50 @@
from __future__ import unicode_literals
import json
from django import forms
from django.utils.translation import ugettext_lazy as _
from common.forms import DynamicModelForm
from .classes import QuotaBackend
from .models import Quota
class QuotaBackendSelectionForm(forms.Form):
backend = forms.ChoiceField(choices=(), label=_('Backend'))
def __init__(self, *args, **kwargs):
super(QuotaBackendSelectionForm, self).__init__(*args, **kwargs)
self.fields['backend'].choices = QuotaBackend.as_choices()
class QuotaDynamicForm(DynamicModelForm):
class Meta:
fields = ('enabled', 'backend_data')
model = Quota
widgets = {'backend_data': forms.widgets.HiddenInput}
def __init__(self, *args, **kwargs):
result = super(QuotaDynamicForm, self).__init__(*args, **kwargs)
if self.instance.backend_data:
for key, value in json.loads(self.instance.backend_data).items():
self.fields[key].initial = value
return result
def clean(self):
data = super(QuotaDynamicForm, self).clean()
# Consolidate the dynamic fields into a single JSON field called
# 'backend_data'.
backend_data = {}
for field in self.schema['fields']:
backend_data[field['name']] = data.pop(
field['name'], field.get('default', None)
)
data['backend_data'] = json.dumps(backend_data)
return data

View File

@@ -0,0 +1,13 @@
from __future__ import unicode_literals
from django.apps import apps
def handler_process_signal(sender, **kwargs):
Quota = apps.get_model(app_label='quotas', model_name='Quota')
for quota in Quota.objects.filter(enabled=True):
backend_instance = quota.get_backend_instance()
if backend_instance.sender == sender and kwargs['signal'].__class__ == backend_instance.signal.__class__:
backend_instance.process(**kwargs)

View File

@@ -0,0 +1,38 @@
from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _
from navigation import Link
from .permissions import (
permission_quota_create, permission_quota_delete,
permission_quota_edit, permission_quota_view,
)
def is_not_editable(context):
return not context['object'].editable
link_quota_create = Link(
icon='fa fa-envelope', permissions=(permission_quota_create,),
text=_('Quota create'), view='quotas:quota_backend_selection',
)
link_quota_delete = Link(
args='resolved_object.pk', conditional_disable=is_not_editable,
permissions=(permission_quota_delete,), tags='dangerous', text=_('Delete'),
view='quotas:quota_delete',
)
link_quota_edit = Link(
args='object.pk', conditional_disable=is_not_editable,
permissions=(permission_quota_edit,), text=_('Edit'),
view='quotas:quota_edit',
)
link_quota_list = Link(
icon='fa fa-envelope', permissions=(permission_quota_view,),
text=_('Quotas list'), view='quotas:quota_list',
)
link_quota_setup = Link(
icon='fa fa-dashboard', permissions=(permission_quota_view,),
text=_('Quotas'), view='quotas:quota_list',
)

View File

@@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-08-01 06:26
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Quota',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('backend_path', models.CharField(help_text='The dotted Python path to the backend class.', max_length=255, verbose_name='Backend path')),
('backend_data', models.TextField(blank=True, verbose_name='Backend data')),
('enabled', models.BooleanField(default=True, verbose_name='Enabled')),
],
options={
'verbose_name': 'Quota',
'verbose_name_plural': 'Quotas',
},
),
]

View File

@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-08-01 07:57
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('quotas', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='quota',
name='editable',
field=models.BooleanField(default=True, editable=False, verbose_name='Editable'),
),
]

View File

View File

@@ -0,0 +1,84 @@
from __future__ import unicode_literals
import json
import logging
from django.db import models
from django.utils.encoding import force_text
from django.utils.module_loading import import_string
from django.utils.translation import ugettext_lazy as _
from .handlers import handler_process_signal
logger = logging.getLogger(__name__)
class Quota(models.Model):
backend_path = models.CharField(
max_length=255,
help_text=_('The dotted Python path to the backend class.'),
verbose_name=_('Backend path')
)
backend_data = models.TextField(
blank=True, verbose_name=_('Backend data')
)
enabled = models.BooleanField(default=True, verbose_name=_('Enabled'))
editable = models.BooleanField(
default=True, editable=False, verbose_name=_('Editable')
)
class Meta:
verbose_name = _('Quota')
verbose_name_plural = _('Quotas')
def __str__(self):
return force_text(self.backend_label())
def save(self, *args, **kwargs):
result = super(Quota, self).save(*args, **kwargs)
self.update_receiver()
return result
def backend_display(self):
return self.get_backend_instance().display()
def backend_label(self):
return self.get_backend_instance().label
def backend_usage(self):
return self.get_backend_instance().usage()
def dispatch_uid(self):
return 'quote_{}'.format(self.pk)
def dumps(self, data):
self.backend_data = json.dumps(data)
self.save()
def get_backend_class(self):
return import_string(self.backend_path)
def get_backend_instance(self):
return self.get_backend_class()(**self.loads())
def loads(self):
return json.loads(self.backend_data)
def update_receiver(self):
backend_instance = self.get_backend_instance()
if self.enabled:
backend_instance.signal.disconnect(
dispatch_uid=self.dispatch_uid(),
sender=backend_instance.sender
)
backend_instance.signal.connect(
handler_process_signal,
dispatch_uid=self.dispatch_uid(),
sender=backend_instance.sender
)
else:
backend_instance.signal.disconnect(
dispatch_uid=self.dispatch_uid(),
sender=backend_instance.sender
)

View File

@@ -0,0 +1,20 @@
from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from permissions import PermissionNamespace
namespace = PermissionNamespace('quotas', _('Quotas'))
permission_quota_create = namespace.add_permission(
name='quota_create', label=_('Create a quota')
)
permission_quota_delete = namespace.add_permission(
name='quota_delete', label=_('Delete a quota')
)
permission_quota_edit = namespace.add_permission(
name='quota_edit', label=_('Edit a quota')
)
permission_quota_view = namespace.add_permission(
name='quota_view', label=_('View a quota')
)

View File

@@ -0,0 +1,211 @@
from __future__ import unicode_literals
from django.contrib.auth import get_user_model
from django.contrib.contenttypes.models import ContentType
from django.db.models.signals import pre_save
from django.template.defaultfilters import filesizeformat
from django.utils.translation import ugettext_lazy as _
from actstream.models import actor_stream
from documents.events import event_document_create, event_document_new_version
from documents.models import Document, DocumentVersion
from .classes import QuotaBackend
from .exceptions import QuotaExceeded
__all__ = ('DocumentStorageQuota', 'DocumentCountQuota',)
class DocumentCountQuota(QuotaBackend):
fields = (
{
'name': 'documents_limit', 'label': _('Documents limit'),
'class': 'django.forms.IntegerField',
'help_text': _('Maximum number of documents')
},
)
label = _('Document count')
sender = Document
signal = pre_save
def __init__(self, documents_limit):
self.documents_limit = documents_limit
def _allowed(self):
return self.documents_limit
def _usage(self, **kwargs):
return Document.passthrough.all().count()
def display(self):
return _(
'Maximum document count: %(total_documents)s'
) % {
'total_documents': self._allowed(),
}
def process(self, **kwargs):
if self._usage() > self._allowed():
raise QuotaExceeded('Document count exceeded')
def usage(self):
return _('%(usage)s out of %(allowed)s') % {
'usage': self._usage(),
'allowed': self._allowed()
}
class DocumentStorageQuota(QuotaBackend):
fields = (
{
'name': 'storage_size', 'label': _('Storage size'),
'class': 'django.forms.FloatField',
'help_text': _('Total storage usage in megabytes (MB)')
},
)
label = _('Document storage')
sender = Document
signal = pre_save
def __init__(self, storage_size):
self.storage_size = storage_size
def _allowed(self):
return self.storage_size * 1024 * 1024
def _usage(self, **kwargs):
total_usage = 0
for document_version in DocumentVersion.objects.all():
if document_version.exists():
total_usage += document_version.file.size
return total_usage
def display(self):
return _(
'Maximum storage usage: %(formatted_file_size)s (%(raw_file_size)s MB)'
) % {
'formatted_file_size': filesizeformat(self._allowed()),
'raw_file_size': self.storage_size
}
def process(self, **kwargs):
if self._usage() > self.storage_size * 1024 * 1024:
raise QuotaExceeded('Storage usage exceeded')
def usage(self):
return _('%(usage)s out of %(allowed)s') % {
'usage': filesizeformat(self._usage()),
'allowed': filesizeformat(self._allowed())
}
class UserDocumentCountQuota(QuotaBackend):
fields = (
{
'name': 'username', 'label': _('Username'),
'class': 'django.forms.CharField', 'kwargs': {
'max_length': 255
}, 'help_text': _(
'Username of the user to which the quota will be applied'
)
},
{
'name': 'documents_limit', 'label': _('Documents limit'),
'class': 'django.forms.IntegerField',
'help_text': _('Maximum number of documents')
},
)
label = _('User document count')
sender = Document
signal = pre_save
def __init__(self, documents_limit, username):
self.documents_limit = documents_limit
self.username = username
def _allowed(self):
return self.documents_limit
def _usage(self, **kwargs):
user = get_user_model().objects.get(username=self.username)
return actor_stream(user).filter(verb=event_document_create.id).count()
def display(self):
user = get_user_model().objects.get(username=self.username)
return _(
'Maximum document count: %(total_documents)s, for user: %(user)s'
) % {
'total_documents': self._allowed(),
'user': user.get_full_name() or user
}
def process(self, **kwargs):
if self._usage() > self._allowed():
raise QuotaExceeded('Document count exceeded')
def usage(self):
return _('%(usage)s out of %(allowed)s') % {
'usage': self._usage(),
'allowed': self._allowed()
}
###
class UserDocumentStorageQuota(QuotaBackend):
fields = (
{
'name': 'username', 'label': _('Username'),
'class': 'django.forms.CharField', 'kwargs': {
'max_length': 255
}, 'help_text': _(
'Username of the user to which the quota will be applied'
)
},
{
'name': 'storage_size', 'label': _('Storage size'),
'class': 'django.forms.FloatField',
'help_text': _('Total storage usage in megabytes (MB)')
},
)
label = _('User document storage')
sender = Document
signal = pre_save
def __init__(self, storage_size, username):
self.storage_size = storage_size
self.username = username
def _allowed(self):
return self.storage_size * 1024 * 1024
def _usage(self, **kwargs):
total_usage = 0
user = get_user_model().objects.get(username=self.username)
content_type = ContentType.objects.get_for_model(model=user)
for document_version in DocumentVersion.objects.filter(target_actions__actor_object_id=1, target_actions__actor_content_type=content_type, target_actions__verb=event_document_new_version.id):
if document_version.exists():
total_usage += document_version.file.size
return total_usage
def display(self):
user = get_user_model().objects.get(username=self.username)
return _(
'Maximum storage usage: %(formatted_file_size)s (%(raw_file_size)s MB), for user %(user)s'
) % {
'formatted_file_size': filesizeformat(self._allowed()),
'raw_file_size': self.storage_size,
'user': user.get_full_name() or user
}
def process(self, **kwargs):
if self._usage() > self._allowed():
raise QuotaExceeded('Document count exceeded')
def usage(self):
return _('%(usage)s out of %(allowed)s') % {
'usage': filesizeformat(self._usage()),
'allowed': filesizeformat(self._allowed())
}

32
mayan/apps/quotas/urls.py Normal file
View File

@@ -0,0 +1,32 @@
from __future__ import unicode_literals
from django.conf.urls import url
from .views import (
QuotaBackendSelectionView, QuotaCreateView, QuotaDeleteView,
QuotaEditView, QuotaListView
)
urlpatterns = [
url(
r'^quotas/backend/selection/$',
QuotaBackendSelectionView.as_view(),
name='quota_backend_selection'
),
url(
r'^quotas/(?P<class_path>[a-zA-Z0-9_.]+)/create/$',
QuotaCreateView.as_view(), name='quota_create'
),
url(
r'^quotas/(?P<pk>\d+)/delete/$', QuotaDeleteView.as_view(),
name='quota_delete'
),
url(
r'^quotas/(?P<pk>\d+)/edit/$', QuotaEditView.as_view(),
name='quota_edit'
),
url(
r'^quotas/$', QuotaListView.as_view(),
name='quota_list'
),
]

108
mayan/apps/quotas/views.py Normal file
View File

@@ -0,0 +1,108 @@
from __future__ import absolute_import, unicode_literals
from django.http import Http404, HttpResponseRedirect
from django.urls import reverse, reverse_lazy
from django.utils.translation import ugettext_lazy as _
from common.generics import (
FormView, SingleObjectDeleteView, SingleObjectDynamicFormCreateView,
SingleObjectDynamicFormEditView, SingleObjectListView
)
from .classes import QuotaBackend
from .forms import QuotaBackendSelectionForm, QuotaDynamicForm
from .models import Quota
from .permissions import (
permission_quota_create, permission_quota_delete,
permission_quota_edit, permission_quota_view
)
class QuotaBackendSelectionView(FormView):
extra_context = {
'title': _('New quota backend selection'),
}
form_class = QuotaBackendSelectionForm
view_permission = permission_quota_create
def form_valid(self, form):
backend = form.cleaned_data['backend']
return HttpResponseRedirect(
reverse('quotas:quota_create', args=(backend,),)
)
class QuotaCreateView(SingleObjectDynamicFormCreateView):
form_class = QuotaDynamicForm
post_action_redirect = reverse_lazy('quotas:quota_list')
view_permission = permission_quota_create
def get_backend(self):
try:
return QuotaBackend.get(name=self.kwargs['class_path'])
except KeyError:
raise Http404(
'{} class not found'.format(self.kwargs['class_path'])
)
def get_extra_context(self):
return {
'title': _(
'Create a "%s" quota'
) % self.get_backend().label,
}
def get_form_schema(self):
return {
'fields': self.get_backend().fields,
'widgets': getattr(self.get_backend(), 'widgets', {})
}
def get_instance_extra_data(self):
return {'backend_path': self.kwargs['class_path']}
class QuotaDeleteView(SingleObjectDeleteView):
object_permission = permission_quota_delete
post_action_redirect = reverse_lazy('quotas:quota_list')
def get_extra_context(self):
return {
'title': _('Delete quota: %s') % self.get_object(),
}
def get_queryset(self):
return Quota.objects.filter(editable=True)
class QuotaEditView(SingleObjectDynamicFormEditView):
form_class = QuotaDynamicForm
object_permission = permission_quota_edit
def form_valid(self, form):
return super(QuotaEditView, self).form_valid(form)
def get_extra_context(self):
return {
'title': _('Edit quota: %s') % self.get_object(),
}
def get_form_schema(self):
return {
'fields': self.get_object().get_backend_class().fields,
'widgets': getattr(
self.get_object().get_backend_class(), 'widgets', {}
)
}
def get_queryset(self):
return Quota.objects.filter(editable=True)
class QuotaListView(SingleObjectListView):
extra_context = {
'hide_object': True,
'title': _('Quotas'),
}
model = Quota
object_permission = permission_quota_view

View File

@@ -97,6 +97,7 @@ INSTALLED_APPS = (
'mirroring',
'motd',
'ocr',
'quotas',
'rest_api',
'sources',
'statistics',