diff --git a/HISTORY.rst b/HISTORY.rst index fece980b29..1e09721b10 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,42 @@ Importer branch - Add support for source column exclusion. - Backport workflow context support. - Backport workflow transitions field support. +- Backport workflow email action. +- Backport individual index rebuild support. +- Rename the installjavascript command to installdependencies. +- Remove database conversion command. +- Remove support for quoted configuration entries. Support unquoted, + nested dictionaries in the configuration. Requires manual + update of existing config.yml files. +- Support user specified locations for the configuration file with the + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), + and CONFIGURATION_LAST_GOOD_FILEPATH + (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. +- Move bootstrapped settings code to their own module in the smart_settings + apps. +- Remove individual database configuration options. All database + configuration is now done using MAYAN_DATABASES to mirror Django way of + doing atabase etup. +- Added support for YAML encoded environment variables to the platform + templates apps. +- Move YAML code to its own module. +- Move Django and Celery settings. +- Backport FakeStorageSubclass from versions/next. +- Remove django-environ. +- Support checking in and out multiple documents. +- Remove encapsulate helper. +- Add support for menu inheritance. +- Emphasize source column labels. + +3.2.6 (2019-07-10) +================== +* Remove the smart settings app * import. +* Encode settings YAML before hashing. +* Fix document icon used in the workflow runtime links. +* Add trashed date time label. +* Fix thumbnail generation issue. GitLab issue #637. + Thanks to Giacomo Cariello (@giacomocariello) for the report + and the merge request fixing the issue. 3.2.5 (2019-07-05) ================== diff --git a/contrib/scripts/install/development.sh b/contrib/scripts/install/development.sh deleted file mode 100644 index a8c9497c46..0000000000 --- a/contrib/scripts/install/development.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -INSTALLATION_DIRECTORY=/home/vagrant/mayan-edms/ -DB_NAME=mayan_edms -DB_PASSWORD=test123 - -cat << EOF | sudo tee -a /etc/motd.tail -**********************************sudo apt - -Mayan EDMS Vagrant Development Box - -********************************** -EOF - -# Update sources -echo -e "\n -> Running apt-get update & upgrade \n" -sudo apt-get -qq update -sudo apt-get -y upgrade - -echo -e "\n -> Installing core binaries \n" -sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice - -echo -e "\n -> Cloning development branch of repository \n" -git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY -cd $INSTALLATION_DIRECTORY -git checkout development -git reset HEAD --hard - -echo -e "\n -> Setting up virtual env \n" -virtualenv venv -source venv/bin/activate - -echo -e "\n -> Installing python dependencies \n" -pip install -r requirements.txt - -echo -e "\n -> Running Mayan EDMS initial setup \n" -./manage.py initialsetup - -echo -e "\n -> Installing Redis server \n" -sudo apt-get install -y redis-server -pip install redis - -echo -e "\n -> Installing testing software \n" -pip install coverage - -echo -e "\n -> Installing MySQL \n" -sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '$DB_PASSWORD -sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '$DB_PASSWORD -sudo apt-get install -y mysql-server libmysqlclient-dev -# Create a passwordless root and travis users -mysql -u root -p$DB_PASSWORD -e "SET PASSWORD = PASSWORD('');" -mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';GRANT ALL PRIVILEGES ON * . * TO 'travis'@'localhost';FLUSH PRIVILEGES;" -mysql -u travis -e "CREATE DATABASE $DB_NAME;" -pip install mysql-python - -echo -e "\n -> Installing PostgreSQL \n" -sudo apt-get install -y postgresql postgresql-server-dev-all -sudo -u postgres psql -c 'create database mayan_edms;' -U postgres -sudo cat > /etc/postgresql/9.3/main/pg_hba.conf << EOF -local all postgres trust - -# TYPE DATABASE USER ADDRESS METHOD - -# "local" is for Unix domain socket connections only -local all all peer -# IPv4 local connections: -host all all 127.0.0.1/32 md5 -# IPv6 local connections: -host all all ::1/128 md5 -EOF - -pip install -q psycopg2 diff --git a/contrib/scripts/install/dialog.sh b/contrib/scripts/install/dialog.sh deleted file mode 100644 index e7d929639c..0000000000 --- a/contrib/scripts/install/dialog.sh +++ /dev/null @@ -1,1733 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2017 Igor Pecovnik, igor.pecovnik@gma**.com -# -# This file is licensed under the terms of the GNU General Public -# License version 2. This program is licensed "as is" without any -# warranty of any kind, whether express or implied. - -# Functions: -# check_status -# choose_webserver -# server_conf -# install_packet -# alive_port -# alive_process -# install_basic -# create_ispconfig_configuration -# install_cups -# install_samba -# install_omv -# install_tvheadend -# install_urbackup -# install_transmission -# install_transmission_seed_armbian_torrents -# install_syncthing -# install_vpn_server -# install_vpn_client -# install_DashNTP -# install_MySQL -# install_MySQLDovecot -# install_Virus -# install_hhvm -# install_phpmyadmin -# install_apache -# install_nginx -# install_PureFTPD -# install_Bind -# install_Stats -# install_Jailkit -# install_Fail2BanDovecot -# install_Fail2BanRulesDovecot -# install_ISPConfig -# check_if_installed - -# -# load functions, local first -# -if [[ -f debian-config-jobs ]]; then source debian-config-jobs; - elif [[ -f /usr/lib/armbian-config/jobs.sh ]]; then source /usr/lib/armbian-config/jobs.sh; - else exit 1; -fi - -if [[ -f debian-config-submenu ]]; then source debian-config-submenu; - elif [[ -f /usr/lib/armbian-config/submenu.sh ]]; then source /usr/lib/armbian-config/submenu.sh; - else exit 1; -fi - -if [[ -f debian-config-functions ]]; then source debian-config-functions; - elif [[ -f /usr/lib/armbian-config/functions.sh ]]; then source /usr/lib/armbian-config/functions.sh; - else exit 1; -fi - -if [[ -f debian-config-functions-network ]]; then source debian-config-functions-network; - elif [[ -f /usr/lib/armbian-config/functions-network.sh ]]; then source /usr/lib/armbian-config/functions-network.sh; - else exit 1; -fi - - - - -# -# not sure if needed -# -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - - - - -function check_status -{ -# -# Check if service is already installed and show it's status -# - -dialog --backtitle "$BACKTITLE" --title "Please wait" --infobox "\nLoading install info ... " 5 28 -LIST=() -LIST_CONST=3 - -# Samba -SAMBA_STATUS="$(check_if_installed samba && echo "on" || echo "off" )" -alive_port "Windows compatible file sharing" "445" "boolean" -LIST+=( "Samba" "$DESCRIPTION" "$SAMBA_STATUS" ) - -# CUPS -CUPS_STATUS="$(check_if_installed cups && echo "on" || echo "off" )" -alive_port "Common UNIX Printing System (CUPS)" "631" "boolean" -LIST+=( "CUPS" "$DESCRIPTION" "$CUPS_STATUS" ) - -# TV headend -TVHEADEND_STATUS="$(check_if_installed tvheadend && echo "on" || echo "off" )" -alive_port "TV streaming server" "9981" -LIST+=( "TV headend" "$DESCRIPTION" "$TVHEADEND_STATUS" ) - -# Synthing -SYNCTHING_STATUS="$([[ -d /usr/bin/syncthing ]] && echo "on" || echo "off" )" -alive_port "Personal cloud @syncthing.net" "8384" "boolean" -LIST+=( "Syncthing" "$DESCRIPTION" "$SYNCTHING_STATUS" ) - -# Mayan EDMS -MAYAN_STATUS="$( [[ -d /opt/mayan ]] && echo "on" || echo "off" )" -alive_port "Electronic Document Management System" "8000" "boolean" -LIST+=( "Mayan EDMS" "$DESCRIPTION" "$MAYAN_STATUS" ) - -# Exagear -if [[ "$(check_if_installed xserver-xorg && echo "on")" == "on" && "$family" == "Ubuntu" ]]; then - EXAGEAR_STATUS="$(check_if_installed exagear-armbian && echo "on" || echo "off" )" - LIST+=( "ExaGear" "32bit x86 Linux/Windows emulator trial" "$EXAGEAR_STATUS" ) -fi - -if [[ "$(dpkg --print-architecture)" == "armhf" || "$(dpkg --print-architecture)" == "amd64" ]]; then - LIST_CONST=2 - # vpn server - VPN_SERVER_STATUS="$([[ -d /usr/local/vpnserver ]] && echo "on" || echo "off" )" - LIST+=( "VPN server" "Softether VPN server" "$VPN_SERVER_STATUS" ) - # vpn client - VPN_CLIENT_STATUS="$([[ -d /usr/local/vpnclient ]] && echo "on" || echo "off" )" - LIST+=( "VPN client" "Softether VPN client" "$VPN_CLIENT_STATUS" ) -fi -# NCP -NCP_STATUS="$( [[ -d /var/www/nextcloud ]] && echo "on" || echo "off" )" -[[ "$family" != "Ubuntu" ]] && LIST+=( "NCP" "Nextcloud personal cloud" "$NCP_STATUS" ) -# OMV -OMV_STATUS="$(check_if_installed openmediavault && echo "on" || echo "off" )" -[[ "$family" != "Ubuntu" ]] && LIST+=( "OMV" "OpenMediaVault NAS solution" "$OMV_STATUS" ) && LIST_CONST=3 - -# Plex media server -PLEX_STATUS="$((check_if_installed plexmediaserver || check_if_installed plexmediaserver-installer) && echo "on" || echo "off" )" -alive_port "Plex media server" "32400" -LIST+=( "Plex" "$DESCRIPTION" "$PLEX_STATUS" ) - -# Radarr -RADARR_STATUS="$([[ -d /opt/Radarr ]] && echo "on" || echo "off" )" -alive_port "Movies downloading server" "7878" -LIST+=( "Radarr" "$DESCRIPTION" "$RADARR_STATUS" ) - -# Sonarr -SONARR_STATUS="$([[ -d /opt/NzbDrone ]] && echo "on" || echo "off" )" -alive_port "TV shows downloading server" "8989" -LIST+=( "Sonarr" "$DESCRIPTION" "$SONARR_STATUS" ) - -# MINIdlna -MINIDLNA_STATUS="$(check_if_installed minidlna && echo "on" || echo "off" )" -alive_port "Lightweight DLNA/UPnP-AV server" "8200" "boolean" -LIST+=( "Minidlna" "$DESCRIPTION" "$MINIDLNA_STATUS" ) - -# Pi hole -PI_HOLE_STATUS="$([[ -d /etc/pihole ]] && echo "on" || echo "off" )" -alive_process "Ad blocker" "pihole-FTL" -LIST+=( "Pi hole" "$DESCRIPTION" "$PI_HOLE_STATUS" ) - -# Transmission -TRANSMISSION_STATUS="$(check_if_installed transmission-daemon && echo "on" || echo "off" )" -alive_port "Torrent download server" "9091" "boolean" -LIST+=( "Transmission" "$DESCRIPTION" "$TRANSMISSION_STATUS" ) - - -# UrBackup -URBACKUP_STATUS="$((check_if_installed urbackup-server || check_if_installed urbackup-server-dbg) && echo "on" || echo "off" )" -alive_port "Client/server backup system" "51413" "boolean" -LIST+=( "UrBackup" "$DESCRIPTION" "$URBACKUP_STATUS" ) - - -# ISPconfig -ISPCONFIG_STATUS="$([[ -d /usr/local/ispconfig ]] && echo "on" || echo "off" )" -LIST+=( "ISPConfig" "SMTP mail, IMAP, POP3 & LAMP/LEMP web server" "$ISPCONFIG_STATUS" ) -} - - - - -function choose_webserver -{ -# -# Target web server selection -# -check_if_installed openmediavault -case $? in - 0) - # OMV installed, prevent switching from nginx to apache which would trash OMV installation - server="nginx" - ;; - *) - dialog --title "Choose a webserver" --backtitle "$BACKTITLE" --yes-label "Apache" --no-label "Nginx" \ - --yesno "\nChoose a web server which you are familiar with. They both work almost the same." 8 70 - response=$? - case $response in - 0) server="apache";; - 1) server="nginx";; - 255) exit;; - esac - ;; -esac -} - - - - -function server_conf -{ -# -# Add some reqired date for installation -# -exec 3>&1 -dialog --title "Server configuration" --separate-widget $'\n' --ok-label "Install" --backtitle "$BACKTITLE" \ ---form "\nPlease fill out this form:\n " \ -12 70 0 \ -"Your FQDN for $serverip:" 1 1 "$hostnamefqdn" 1 31 32 0 \ -"Mysql root password:" 2 1 "$mysql_pass" 2 31 32 0 \ -2>&1 1>&3 | { - -read -r hostnamefqdn -read -r mysql_pass -echo $mysql_pass > ${TEMP_DIR}/mysql_pass -echo $hostnamefqdn > ${TEMP_DIR}/hostnamefqdn -# end -} -exec 3>&- -# read variables back -read MYSQL_PASS < ${TEMP_DIR}/mysql_pass -read HOSTNAMEFQDN < ${TEMP_DIR}/hostnamefqdn -} - - - - -install_packet () -{ -# -# Install missing packets -# -i=0 -j=1 -IFS=" " -declare -a PACKETS=($1) -#skupaj=$(apt-get -s -y -qq install $1 | wc -l) -skupaj=${#PACKETS[@]} -while [[ $i -lt $skupaj ]]; do -procent=$(echo "scale=2;($j/$skupaj)*100"|bc) - x=${PACKETS[$i]} - if [ $(dpkg-query -W -f='${Status}' $x 2>/dev/null | grep -c "ok installed") -eq 0 ]; then - printf '%.0f\n' $procent | dialog \ - --backtitle "$BACKTITLE" \ - --title "Installing" \ - --gauge "\n$2\n\n$x" 10 70 - if [ "$(DEBIAN_FRONTEND=noninteractive apt-get -qq -y install $x >${TEMP_DIR}/install.log 2>&1 || echo 'Installation failed' \ - | grep 'Installation failed')" != "" ]; then - echo -e "[\e[0;31m error \x1B[0m] Installation failed" - tail ${TEMP_DIR}/install.log - exit - fi - fi - i=$[$i+1] - j=$[$j+1] -done -echo "" -} - - -alive_port () -{ -# -# Displays URL to the service $1 on port $2 or just that is active if $3 = boolean -# -DEFAULT_ADAPTER=$(ip -4 route ls | grep default | grep -Po '(?<=dev )(\S+)') -LOCALIPADD=$(ip -4 addr show dev $DEFAULT_ADAPTER | awk '/inet/ {print $2}' | cut -d'/' -f1) -if [[ -n $(netstat -lnt | awk '$6 == "LISTEN" && $4 ~ ".'$2'"') ]]; then - if [[ $3 == boolean ]]; then - DESCRIPTION="$1 is \Z1active\Z0"; - else - DESCRIPTION="Active on http://${LOCALIPADD}:\Z1$2\Z0"; - fi -else -DESCRIPTION="$1"; -fi -} - - - -alive_process () -{ -# -# check if process name $2 is running. Display it's name $1 or $1 is active if active -# -if pgrep -x "$2" > /dev/null 2>&1; then DESCRIPTION="$1 is \Z1active\Z0"; else DESCRIPTION="$1"; fi -} - - - - - - - -install_basic (){ -# -# Set hostname, FQDN, add to sources list -# -IFS=" " -set ${HOSTNAMEFQDN//./ } -HOSTNAMESHORT="$1" -cp /etc/hosts /etc/hosts.backup -cp /etc/hostname /etc/hostname.backup -# create new -echo "127.0.0.1 localhost.localdomain localhost" > /etc/hosts -echo "${serverIP} ${HOSTNAMEFQDN} ${HOSTNAMESHORT} #ispconfig " >> /etc/hosts -echo "$HOSTNAMESHORT" > /etc/hostname -/etc/init.d/hostname.sh start >/dev/null 2>&1 -hostnamectl set-hostname $HOSTNAMESHORT -if [[ $family == "Ubuntu" ]]; then - # set hostname in Ubuntu - hostnamectl set-hostname $HOSTNAMESHORT - # disable AppArmor - if [[ -n $(service apparmor status | grep -w active | grep -w running) ]]; then - service apparmor stop - update-rc.d -f apparmor remove - apt-get -y -qq remove apparmor apparmor-utils - fi -else - grep -q "contrib" /etc/apt/sources.list || sed -i 's|main|main contrib|' /etc/apt/sources.list - grep -q "non-free" /etc/apt/sources.list || sed -i 's|contrib|contrib non-free|' /etc/apt/sources.list - grep -q "deb http://ftp.debian.org/debian jessie-backports main" /etc/apt/sources.list || echo "deb http://ftp.debian.org/debian jessie-backports main" >> /etc/apt/sources.list - debconf-apt-progress -- apt-get update -fi -} - - - - -create_ispconfig_configuration (){ -# -# ISPConfig autoconfiguration -# -cat > ${TEMP_DIR}/isp.conf.php < -EOF -} - - - -install_cups () -{ -# -# Install printer system -# -debconf-apt-progress -- apt-get -y install cups lpr cups-filters -# cups-filters if jessie -sed -e 's/Listen localhost:631/Listen 631/g' -i /etc/cups/cupsd.conf -sed -e 's//\nallow $SUBNET/g' -i /etc/cups/cupsd.conf -sed -e 's//\nallow $SUBNET/g' -i /etc/cups/cupsd.conf -sed -e 's//\nallow $SUBNET/g' -i /etc/cups/cupsd.conf -service cups restart -service samba restart | service smbd restart >/dev/null 2>&1 -} - - - - -install_samba () -{ -# -# install Samba file sharing -# -local SECTION="Samba" -SMBUSER=$(whiptail --inputbox "What is your samba username?" 8 78 $SMBUSER --title "$SECTION" 3>&1 1>&2 2>&3) -exitstatus=$?; if [ $exitstatus = 1 ]; then exit 1; fi -SMBPASS=$(whiptail --inputbox "What is your samba password?" 8 78 $SMBPASS --title "$SECTION" 3>&1 1>&2 2>&3) -exitstatus=$?; if [ $exitstatus = 1 ]; then exit 1; fi -SMBGROUP=$(whiptail --inputbox "What is your samba group?" 8 78 $SMBGROUP --title "$SECTION" 3>&1 1>&2 2>&3) -exitstatus=$?; if [ $exitstatus = 1 ]; then exit 1; fi -# -debconf-apt-progress -- apt-get -y install samba samba-common-bin samba-vfs-modules -useradd $SMBUSER -echo -ne "$SMBPASS\n$SMBPASS\n" | passwd $SMBUSER >/dev/null 2>&1 -echo -ne "$SMBPASS\n$SMBPASS\n" | smbpasswd -a -s $SMBUSER >/dev/null 2>&1 -service samba stop | service smbd stop >/dev/null 2>&1 -cp /etc/samba/smb.conf /etc/samba/smb.conf.stock -cat > /etc/samba/smb.conf.tmp << EOF -[global] - workgroup = SMBGROUP - server string = %h server - hosts allow = SUBNET - log file = /var/log/samba/log.%m - max log size = 1000 - syslog = 0 - panic action = /usr/share/samba/panic-action %d - load printers = yes - printing = cups - printcap name = cups - min receivefile size = 16384 - write cache size = 524288 - getwd cache = yes - socket options = TCP_NODELAY IPTOS_LOWDELAY - -[printers] - comment = All Printers - path = /var/spool/samba - browseable = no - public = yes - guest ok = yes - writable = no - printable = yes - printer admin = SMBUSER - -[print$] - comment = Printer Drivers - path = /etc/samba/drivers - browseable = yes - guest ok = no - read only = yes - write list = SMBUSER - -[ext] - comment = Storage - path = /ext - writable = yes - public = no - valid users = SMBUSER - force create mode = 0644 -EOF -sed -i "s/SMBGROUP/$SMBGROUP/" /etc/samba/smb.conf.tmp -sed -i "s/SMBUSER/$SMBUSER/" /etc/samba/smb.conf.tmp -sed -i "s/SUBNET/$SUBNET/" /etc/samba/smb.conf.tmp -dialog --backtitle "$BACKTITLE" --title "Review samba configuration" --no-collapse --editbox /etc/samba/smb.conf.tmp 30 0 2> /etc/samba/smb.conf.tmp.out -if [[ $? = 0 ]]; then - mv /etc/samba/smb.conf.tmp.out /etc/samba/smb.conf - install -m 755 -g $SMBUSER -o $SMBUSER -d /ext - service service smbd stop >/dev/null 2>&1 - sleep 3 - service service smbd start >/dev/null 2>&1 -fi -} - -install_ncp (){ - curl -sSL https://raw.githubusercontent.com/nextcloud/nextcloudpi/master/install.sh | bash -} - -install_omv (){ -# -# On Debian install OpenMediaVault 3 (Jessie) or 4 (Stretch) -# -# TODO: Some OMV packages lack authentication - -if [[ "$family" == "Ubuntu" ]]; then - dialog --backtitle "$BACKTITLE" --title "Dependencies not met" --msgbox "\nOpenMediaVault can only be installed on Debian." 7 52 - sleep 5 - exit 1 -fi - -case $distribution in - jessie) - OMV_Name="erasmus" - OMV_EXTRAS_URL="https://github.com/OpenMediaVault-Plugin-Developers/packages/raw/master/openmediavault-omvextrasorg_latest_all3.deb" - ;; - stretch) - OMV_Name="arrakis" - OMV_EXTRAS_URL="https://github.com/OpenMediaVault-Plugin-Developers/packages/raw/master/openmediavault-omvextrasorg_latest_all4.deb" - ;; -esac - -systemctl status log2ram >/dev/null 2>&1 && (systemctl stop log2ram ; systemctl disable log2ram >/dev/null 2>&1; rm /etc/cron.daily/log2ram) -export APT_LISTCHANGES_FRONTEND=none -if [ -f /etc/armbian-release ]; then - . /etc/armbian-release -else - sed -i "s/^# en_US.UTF-8/en_US.UTF-8/" /etc/locale.gen - locale-gen -fi - -# preserve cpufrequtils settings: -if [ -f /etc/default/cpufrequtils ]; then - . /etc/default/cpufrequtils -fi - -cat > /etc/apt/sources.list.d/openmediavault.list << EOF -deb https://openmediavault.github.io/packages/ ${OMV_Name} main - -## Uncomment the following line to add software from the proposed repository. -deb https://openmediavault.github.io/packages/ ${OMV_Name}-proposed main - -## This software is not part of OpenMediaVault, but is offered by third-party -## developers as a service to OpenMediaVault users. -# deb https://openmediavault.github.io/packages/ ${OMV_Name} partner -EOF - -debconf-apt-progress -- apt-get update - -read HOSTNAME /dev/null | awk -F" " '/additional disk space will be used/ {print $4}') -SPACE_AVAIL=$(df -k / | awk -F" " '/\/$/ {printf ("%0.0f",$4/1200); }') -if [ ${SPACE_AVAIL} -lt ${SPACE_NEEDED} ]; then - dialog --backtitle "$BACKTITLE" --title "No space left on device" --msgbox "\nOpenMediaVault needs ${SPACE_NEEDED} MB for installation while only ${SPACE_AVAIL} MB are available." 7 52 - exit 1 -fi -apt-get --allow-unauthenticated install openmediavault-keyring -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 7AA630A1EDEE7D73 -debconf-apt-progress -- apt-get -y --allow-unauthenticated --fix-missing --no-install-recommends \ - -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install openmediavault postfix dirmngr -FILE="${TEMP_DIR}/omv_extras.deb"; wget "$OMV_EXTRAS_URL" -qO $FILE && dpkg -i $FILE ; rm $FILE -# /usr/sbin/omv-update -debconf-apt-progress -- apt-get update -debconf-apt-progress -- apt-get --yes --force-yes --fix-missing --auto-remove --allow-unauthenticated \ - --show-upgraded --option DPkg::Options::="--force-confold" dist-upgrade - -# Install flashmemory plugin and netatalk by default, use nice logo for the latter, -# disable OMV monitoring by default -. /usr/share/openmediavault/scripts/helper-functions -debconf-apt-progress -- apt-get -y --fix-missing --no-install-recommends --auto-remove install openmediavault-flashmemory openmediavault-netatalk -AFP_Options="mimic model = Macmini" -SMB_Options="min receivefile size = 16384\nwrite cache size = 524288\ngetwd cache = yes\nsocket options = TCP_NODELAY IPTOS_LOWDELAY" -xmlstarlet ed -L -u "/config/services/afp/extraoptions" -v "$(echo -e "${AFP_Options}")" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/smb/extraoptions" -v "$(echo -e "${SMB_Options}")" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/flashmemory/enable" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/ssh/enable" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/ssh/permitrootlogin" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/system/time/ntp/enable" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/system/time/timezone" -v "${TZ}" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/system/network/dns/hostname" -v "${HOSTNAME}" ${OMV_CONFIG_FILE} -/usr/sbin/omv-rpc -u admin "perfstats" "set" '{"enable":false}' -/usr/sbin/omv-rpc -u admin "config" "applyChanges" '{ "modules": ["monit","rrdcached","collectd"],"force": true }' -sed -i 's|-j /var/lib/rrdcached/journal/ ||' /etc/init.d/rrdcached -/sbin/folder2ram -enablesystemd 2>/dev/null - -# Prevent accidentally destroying board performance by clicking around in OMV UI since -# OMV sets 'powersave' governor when touching 'Power Management' settings. -if [ ! -f /etc/default/cpufrequtils ]; then - DEFAULT_GOV="$(zgrep "^CONFIG_CPU_FREQ_DEFAULT_GOV_" /proc/config.gz 2>/dev/null | sed 's/CONFIG_CPU_FREQ_DEFAULT_GOV_//')" - if [ -n "${DEFAULT_GOV}" ]; then - GOVERNOR=$(cut -f1 -d= <<<"${DEFAULT_GOV}" | tr '[:upper:]' '[:lower:]') - else - GOVERNOR=ondemand - fi - MIN_SPEED="0" - MAX_SPEED="0" -fi -echo -e "OMV_CPUFREQUTILS_GOVERNOR=${GOVERNOR}" >>/etc/default/openmediavault -echo -e "OMV_CPUFREQUTILS_MINSPEED=${MIN_SPEED}" >>/etc/default/openmediavault -echo -e "OMV_CPUFREQUTILS_MAXSPEED=${MAX_SPEED}" >>/etc/default/openmediavault -for i in netatalk samba flashmemory ssh ntp timezone monit rrdcached collectd cpufrequtils ; do - /usr/sbin/omv-mkconf $i -done - -# Hardkernel Cloudshell 1 and 2 fixes, read the whole thread for details: -# https://forum.openmediavault.org/index.php/Thread/17855 -lsusb | grep -q -i "05e3:0735" && sed -i "/exit 0/i echo 20 > /sys/class/block/sda/queue/max_sectors_kb" /etc/rc.local -if [ "X${BOARD}" = "Xodroidxu4" ]; then - HMP_Fix='; taskset -c -p 4-7 $i ' - apt install -y i2c-tools - /usr/sbin/i2cdetect -y 1 | grep -q "60: 60" - if [ $? -eq 0 ]; then - add-apt-repository -y ppa:kyle1117/ppa - sed -i 's/jessie/xenial/' /etc/apt/sources.list.d/kyle1117-ppa-jessie.list - apt install -y -q cloudshell-lcd odroid-cloudshell cloudshell2-fan & - lsusb -v | awk -F"__" '/RANDOM_/ {print $2}' | head -n1 | while read ; do - echo "ATTRS{idVendor}==\"152d\", ATTRS{idProduct}==\"0561\", KERNEL==\"sd*\", ENV{DEVTYPE}==\"disk\", SYMLINK=\"disk/by-id/\$env{ID_BUS}-CloudShell2-${REPLY}-\$env{ID_MODEL}\"" >> /etc/udev/rules.d/99-cloudshell2.rules - echo "ATTRS{idVendor}==\"152d\", ATTRS{idProduct}==\"0561\", KERNEL==\"sd*\", ENV{DEVTYPE}==\"partition\", SYMLINK=\"disk/by-id/\$env{ID_BUS}-CloudShell2-${REPLY}-\$env{ID_MODEL}-part%n\"" >> /etc/udev/rules.d/99-cloudshell2.rules - done - fi -fi - -# Add a cron job to make NAS processes more snappy -systemctl status rsyslog >/dev/null 2>&1 -if [ $? -eq 0 ]; then - echo ':msg, contains, "do ionice -c1" ~' >/etc/rsyslog.d/omv-armbian.conf - systemctl restart rsyslog -fi -echo "* * * * * root for i in \`pgrep \"ftpd|nfsiod|smbd|afpd|cnid\"\` ; do ionice -c1 -p \$i ${HMP_Fix}; done >/dev/null 2>&1" >/etc/cron.d/make_nas_processes_faster -chmod 600 /etc/cron.d/make_nas_processes_faster - -/usr/sbin/omv-initsystem -} - - - - -install_tvheadend () -{ -# -# TVheadend https://tvheadend.org/ unofficial port https://tvheadend.org/boards/5/topics/21528 -# -if [ ! -f /etc/apt/sources.list.d/tvheadend.list ]; then - echo "deb https://dl.bintray.com/tvheadend/deb xenial release-4.2" >> /etc/apt/sources.list.d/tvheadend.list - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 >/dev/null 2>&1 -fi - -if [[ $distribution == "stretch" ]]; then - URL="http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl1.0.0_1.0.1t-1+deb8u8_"$(dpkg --print-architecture)".deb" - fancy_wget "$URL" "-O ${TEMP_DIR}/package.deb" - dpkg -i ${TEMP_DIR}/package.deb >/dev/null 2>&1 - local pkglist="libssl-doc zlib1g-dev tvheadend xmltv-util" -else - local pkglist="libssl-doc libssl1.0.0 zlib1g-dev tvheadend xmltv-util" -fi - -debconf-apt-progress -- apt-get update -debconf-apt-progress -- apt-get -y install $pkglist -} - - - - -install_urbackup () -{ -# -# Client/server backup system https://www.urbackup.org/ -# -if [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then local arch=armhf; else local arch=$(dpkg --print-architecture); fi -PREFIX="http://hndl.urbackup.org/Server/latest/" -URL="http://hndl.urbackup.org/Server/latest/"$(wget -q $PREFIX -O - | html2text -width 120 | grep deb | awk ' { print $3 }' | grep $arch) -fancy_wget "$URL" "-O ${TEMP_DIR}/package.deb" -dpkg -i ${TEMP_DIR}/package.deb >/dev/null 2>&1 -apt-get -yy -f install -} - - - - -install_transmission () -{ -# -# transmission -# -install_packet "debconf-utils unzip build-essential html2text apt-transport-https" "Downloading dependencies" -install_packet "transmission-cli transmission-common transmission-daemon" "Install torrent server" -# systemd workaround -# https://forum.armbian.com/index.php?/topic/4017-programs-does-not-start-automatically-at-boot/ -sed -e 's/exit 0//g' -i /etc/rc.local - cat >> /etc/rc.local <<"EOF" -service transmission-daemon restart -exit 0 -EOF -} - - - -install_transmission_seed_armbian_torrents () -{ -# -# seed our torrents -# -# adjust network buffers if necessary -rmem_recommended=4194304 -wmem_recommended=1048576 -rmem_actual=$(sysctl net.core.rmem_max | awk -F" " '{print $3}') -if [ ${rmem_actual} -lt ${rmem_recommended} ]; then - grep -q net.core.rmem_max /etc/sysctl.conf && \ - sed -i "s/net.core.rmem_max =.*/net.core.rmem_max = ${rmem_recommended}/" /etc/sysctl.conf || \ - echo "net.core.rmem_max = ${rmem_recommended}" >> /etc/sysctl.conf -fi -wmem_actual=$(sysctl net.core.wmem_max | awk -F" " '{print $3}') -if [ ${wmem_actual} -lt ${wmem_recommended} ]; then - grep -q net.core.wmem_max /etc/sysctl.conf && \ - sed -i "s/net.core.wmem_max =.*/net.core.wmem_max = ${wmem_recommended}/" /etc/sysctl.conf || \ - echo "net.core.wmem_max = ${wmem_recommended}" >> /etc/sysctl.conf -fi -/sbin/sysctl -p -# create cron job for daily sync with official Armbian torrents -cat > /etc/cron.daily/seed-armbian-torrent <<"EOF" -#!/bin/bash -# -# armbian torrents auto update -# -# download latest torrent pack -wget -qO- -O ${TEMP_DIR}/armbian-torrents.zip https://dl.armbian.com/torrent/all-torrents.zip -# test zip for corruption -unzip -t ${TEMP_DIR}/armbian-torrents.zip >/dev/null 2>&1 -[[ $? -ne 0 ]] && echo "Error in zip" && exit -# extract zip -unzip -o ${TEMP_DIR}/armbian-torrents.zip -d ${TEMP_DIR}/torrent-tmp >/dev/null 2>&1 -# create list of current active torrents -transmission-remote -n 'transmission:transmission' -l | sed '1d; $d' > ${TEMP_DIR}/torrent-tmp/active.torrents -# loop and add/update torrent files -for f in ${TEMP_DIR}/torrent-tmp/*.torrent; do - transmission-remote -n 'transmission:transmission' -a $f > /dev/null 2>&1 - # remove added from the list - pattern="${f//.torrent}"; pattern="${pattern##*/}"; - sed -i "/$pattern/d" ${TEMP_DIR}/torrent-tmp/active.torrents -done -# remove old armbian torrents -while read i; do - [[ $i == *Armbian_* ]] && transmission-remote -n 'transmission:transmission' -t $(echo "$i" | awk '{print $1}';) --remove-and-delete -done < ${TEMP_DIR}/torrent-tmp/active.torrents -# remove temporally files and direcotories -EOF -chmod +x /etc/cron.daily/seed-armbian-torrent -/etc/cron.daily/seed-armbian-torrent & -} - - - - -install_syncthing () -{ -# -# Install Personal cloud https://syncthing.net/ -# - -if [ "$(dpkg --print-architecture | grep armhf)" == "armhf" ]; then - local filename="linux-arm" -elif [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then - local filename="linux-arm64" -else - local filename="linux-amd64" -fi -mkdir -p /usr/bin/syncthing -wgeturl=$(curl -s "https://api.github.com/repos/syncthing/syncthing/releases" | grep $filename | grep 'browser_download_url' | head -1 | cut -d \" -f 4) -fancy_wget "$wgeturl" "-O ${TEMP_DIR}/syncthing.tgz" -wgeturl=$(curl -s "https://api.github.com/repos/syncthing/syncthing-inotify/releases" | grep $filename | grep 'browser_download_url' | head -1 | cut -d \" -f 4) -fancy_wget "$wgeturl" "-O ${TEMP_DIR}/syncthing-inotify.tgz" -tar xf ${TEMP_DIR}/syncthing.tgz -C ${TEMP_DIR} -tar xf ${TEMP_DIR}/syncthing-inotify.tgz -C /usr/bin -cp -R ${TEMP_DIR}/syncthing-*/syncthing /usr/bin -cp ${TEMP_DIR}/syncthing-*/etc/linux-systemd/system/syncthing* /etc/systemd/system/ -cp /etc/systemd/system/syncthing@.service /etc/systemd/system/syncthing-inotify@.service - -# adjust service for inotify -sed -i "s/^Description=.*/Description=Syncthing Inotify File Watcher/" /etc/systemd/system/syncthing-inotify@.service -sed -i "s/^After=.*/After=network.target syncthing.service/" /etc/systemd/system/syncthing-inotify@.service -sed -i "s/^ExecStart=.*/ExecStart=\/usr\/bin\/syncthing-inotify -logfile=\/var\/log\/syncthing-inotify.log -logflags=3/" /etc/systemd/system/syncthing-inotify@.service -sed -i "/^\[Install\]/a Requires=syncthing.service" /etc/systemd/system/syncthing-inotify@.service - -# increase open file limit -if !(grep -qs "fs.inotify.max_user_watches=204800" "/etc/sysctl.conf");then - echo -e "fs.inotify.max_user_watches=204800" | tee -a /etc/sysctl.conf -fi -add_choose_user -systemctl enable syncthing@${CHOSEN_USER}.service >/dev/null 2>&1 -systemctl start syncthing@${CHOSEN_USER}.service >/dev/null 2>&1 -systemctl enable syncthing-inotify@${CHOSEN_USER}.service >/dev/null 2>&1 -systemctl start syncthing-inotify@${CHOSEN_USER}.service >/dev/null 2>&1 -} - - - - -install_plex_media_server () -{ -# -# Media server -# -if [ "$(dpkg --print-architecture | grep armhf)" == "armhf" ]; then - echo -e "deb [arch=armhf] http://dev2day.de/pms/ stretch main" > /etc/apt/sources.list.d/plex.list - wget -q -O - http://dev2day.de/pms/dev2day-pms.gpg.key | apt-key add - >/dev/null 2>&1 - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install plexmediaserver-installer -elif [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then - echo -e "deb [arch=armhf] http://dev2day.de/pms/ stretch main" > /etc/apt/sources.list.d/plex.list - wget -q -O - http://dev2day.de/pms/dev2day-pms.gpg.key | apt-key add - >/dev/null 2>&1 - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install binutils:armhf plexmediaserver-installer:armhf -else - fancy_wget "https://downloads.plex.tv/plex-media-server/1.12.3.4973-215c28d86/plexmediaserver_1.12.3.4973-215c28d86_amd64.deb" "-O ${TEMP_DIR}/package.deb" - dpkg -i ${TEMP_DIR}/package.deb >/dev/null 2>&1 -fi -} - - - - -install_radarr () -{ -# -# Automatically downloading movies -# -debconf-apt-progress -- apt-get update -debconf-apt-progress -- apt-get -y install mono-devel mediainfo libmono-cil-dev -wgeturl=$(curl -s "https://api.github.com/repos/Radarr/Radarr/releases" | grep 'linux.tar.gz' | grep 'browser_download_url' | head -1 | cut -d \" -f 4) -fancy_wget "$wgeturl" "-O ${TEMP_DIR}/radarr.tgz" -tar xf ${TEMP_DIR}/radarr.tgz -C /opt -cat << _EOF_ > /etc/systemd/system/radarr.service -[Unit] -Description=Radarr Daemon -After=network.target -[Service] -User=root -Type=simple -ExecStart=/usr/bin/mono --debug /opt/Radarr/Radarr.exe -nobrowser -[Install] -WantedBy=multi-user.target -_EOF_ -systemctl enable radarr >/dev/null 2>&1 -systemctl start radarr -} - - - - -install_sonarr () -{ -# -# Automatically downloading TV shows -# -if [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install mono-complete mediainfo - fancy_wget "http://update.sonarr.tv/v2/develop/mono/NzbDrone.develop.tar.gz" "-O ${TEMP_DIR}/sonarr.tgz" - tar xf ${TEMP_DIR}/sonarr.tgz -C /opt -else - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys FDA5DFFC >/dev/null 2>&1 - echo -e "deb https://apt.sonarr.tv/ develop main" > /etc/apt/sources.list.d/sonarr.list - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install nzbdrone -fi -cat << _EOF_ > /etc/systemd/system/sonarr.service -[Unit] -Description=Sonarr (NzbDrone) Daemon -After=network.target -[Service] -User=root -Type=simple -ExecStart=/usr/bin/mono --debug /opt/NzbDrone/NzbDrone.exe -nobrowser -[Install] -WantedBy=multi-user.target -_EOF_ -systemctl enable sonarr >/dev/null 2>&1 -systemctl start sonarr -} - - - - -install_vpn_server () -{ -# -# Script downloads latest stable -# -cd ${TEMP_DIR} -PREFIX="http://www.softether-download.com/files/softether/" -install_packet "debconf-utils unzip build-essential html2text apt-transport-https" "Downloading basic packages" -URL=$(wget -q $PREFIX -O - | html2text | grep rtm | awk ' { print $(NF) }' | tail -1) -SUFIX="${URL/-tree/}" -if [ "$(dpkg --print-architecture | grep armhf)" != "" ]; then -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Server/32bit_-_ARM_EABI/softether-vpnserver-$SUFIX-linux-arm_eabi-32bit.tar.gz" -else -install_packet "gcc-multilib" "Install libraries" -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Server/32bit_-_Intel_x86/softether-vpnserver-$SUFIX-linux-x86-32bit.tar.gz" -fi -wget -q $DLURL -O - | tar -xz -cd vpnserver -make i_read_and_agree_the_license_agreement | dialog --backtitle "$BACKTITLE" --title "Compiling SoftEther VPN" --progressbox $TTY_Y $TTY_X -cd .. -cp -R vpnserver /usr/local -cd /usr/local/vpnserver/ -chmod 600 * -chmod 700 vpncmd -chmod 700 vpnserver -if [[ -d /run/systemd/system/ ]]; then -cat </lib/systemd/system/ethervpn.service -[Unit] -Description=VPN service - -[Service] -Type=oneshot -ExecStart=/usr/local/vpnserver/vpnserver start -ExecStop=/usr/local/vpnserver/vpnserver stop -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target -EOT -systemctl enable ethervpn.service -service ethervpn start - -else - -cat < /etc/init.d/vpnserver -#!/bin/sh -### BEGIN INIT INFO -# Provides: vpnserver -# Required-Start: \$remote_fs \$syslog -# Required-Stop: \$remote_fs \$syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start daemon at boot time -# Description: Enable Softether by daemon. -### END INIT INFO -DAEMON=/usr/local/vpnserver/vpnserver -LOCK=/var/lock/vpnserver -test -x $DAEMON || exit 0 -case "\$1" in -start) -\$DAEMON start -touch \$LOCK -;; -stop) -\$DAEMON stop -rm \$LOCK -;; -restart) -\$DAEMON stop -sleep 3 -\$DAEMON start -;; -*) -echo "Usage: \$0 {start|stop|restart}" -exit 1 -esac -exit 0 -EOT -chmod 755 /etc/init.d/vpnserver -mkdir /var/lock/subsys -update-rc.d vpnserver defaults >> $logfile -/etc/init.d/vpnserver start -fi -} - - - - -install_vpn_client () -{ -# -# Script downloads latest stable -# -cd ${TEMP_DIR} -PREFIX="http://www.softether-download.com/files/softether/" -install_packet "debconf-utils unzip build-essential html2text apt-transport-https" "Downloading basic packages" -URL=$(wget -q $PREFIX -O - | html2text | grep rtm | awk ' { print $(NF) }' | tail -1) -SUFIX="${URL/-tree/}" -if [ "$(dpkg --print-architecture | grep armhf)" != "" ]; then -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Client/32bit_-_ARM_EABI/softether-vpnclient-$SUFIX-linux-arm_eabi-32bit.tar.gz" -else -install_packet "gcc-multilib" "Install libraries" -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Client/32bit_-_Intel_x86/softether-vpnclient-$SUFIX-linux-x86-32bit.tar.gz" -fi -wget -q $DLURL -O - | tar -xz -cd vpnclient -make i_read_and_agree_the_license_agreement | dialog --backtitle "$BACKTITLE" --title "Compiling SoftEther VPN vpnclient" --progressbox $TTY_Y $TTY_X -cd .. -cp -R vpnclient /usr/local -cd /usr/local/vpnclient/ -chmod 600 * -chmod 700 vpncmd -chmod 700 vpnclient -} - - - - -install_DashNTP () -{ -# -# Install DASH and ntp service -# -echo "dash dash/sh boolean false" | debconf-set-selections -dpkg-reconfigure -f noninteractive dash > /dev/null 2>&1 -install_packet "ntp ntpdate" "Install DASH and ntp service" -} - - - - -install_MySQL () -{ -# -# Maria SQL -# -install_packet "mariadb-client mariadb-server" "SQL client and server" -#Allow MySQL to listen on all interfaces -cp /etc/mysql/my.cnf /etc/mysql/my.cnf.backup -[[ -f /etc/mysql/my.cnf ]] && sed -i 's|bind-address.*|#bind-address = 127.0.0.1|' /etc/mysql/my.cnf -[[ -f /etc/mysql/mariadb.conf.d/50-server.cnf ]] && sed -i 's|bind-address.*|#bind-address = 127.0.0.1|' /etc/mysql/mariadb.conf.d/50-server.cnf -SECURE_MYSQL=$(expect -c " -set timeout 3 -spawn mysql_secure_installation -expect \"Enter current password for root (enter for none):\" -send \"\r\" -expect \"root password?\" -send \"y\r\" -expect \"New password:\" -send \"$MYSQL_PASS\r\" -expect \"Re-enter new password:\" -send \"$MYSQL_PASS\r\" -expect \"Remove anonymous users?\" -send \"y\r\" -expect \"Disallow root login remotely?\" -send \"y\r\" -expect \"Remove test database and access to it?\" -send \"y\r\" -expect \"Reload privilege tables now?\" -send \"y\r\" -expect eof -") -# -# Execution mysql_secure_installation -# -echo "${SECURE_MYSQL}" >> /dev/null -# ISP config exception -mkdir -p /etc/mysql/mariadb.conf.d/ -cat > /etc/mysql/mariadb.conf.d/99-ispconfig.cnf<<"EOF" -[mysqld] -sql-mode="NO_ENGINE_SUBSTITUTION" -EOF -service mysql restart >> /dev/null -} - - - - -install_MySQLDovecot () -{ -# -# Install Postfix, Dovecot, Saslauthd, rkhunter, binutils -# -echo "postfix postfix/main_mailer_type select Internet Site" | debconf-set-selections -echo "postfix postfix/mailname string $HOSTNAMEFQDN" | debconf-set-selections -install_packet "postfix postfix-mysql postfix-doc openssl getmail4 rkhunter binutils dovecot-imapd dovecot-pop3d dovecot-mysql \ -dovecot-sieve sudo libsasl2-modules" "postfix, dovecot, saslauthd, rkhunter, binutils" -#Uncommenting some Postfix configuration files -cp /etc/postfix/master.cf /etc/postfix/master.cf.backup -sed -i 's|#submission inet n - - - - smtpd|submission inet n - - - - smtpd|' /etc/postfix/master.cf -sed -i 's|# -o syslog_name=postfix/submission| -o syslog_name=postfix/submission|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_tls_security_level=encrypt| -o smtpd_tls_security_level=encrypt|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_client_restrictions=permit_sasl_authenticated,reject| -o smtpd_client_restrictions=permit_sasl_authenticated,reject|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|#smtps inet n - - - - smtpd|smtps inet n - - - - smtpd|' /etc/postfix/master.cf -sed -i 's|# -o syslog_name=postfix/smtps| -o syslog_name=postfix/smtps|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_tls_wrappermode=yes| -o smtpd_tls_wrappermode=yes|' /etc/postfix/master.cf -service postfix restart >> /dev/null -} - - - - -install_Virus () -{ -# -# Install Amavisd-new, SpamAssassin, And Clamav -# -install_packet "amavisd-new spamassassin clamav clamav-daemon zoo unzip bzip2 arj p7zip unrar-free ripole rpm nomarch lzop \ -cabextract apt-listchanges libnet-ldap-perl libauthen-sasl-perl clamav-docs daemon libio-string-perl libio-socket-ssl-perl \ -libnet-ident-perl zip libnet-dns-perl postgrey" "amavisd, spamassassin, clamav" -sed -i "s/^AllowSupplementaryGroups.*/AllowSupplementaryGroups true/" /etc/clamav/clamd.conf -service spamassassin stop >/dev/null 2>&1 -systemctl disable spamassassin >/dev/null 2>&1 -} - - - - -install_hhvm () -{ -# -# Install HipHop Virtual Machine -# -apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xB4112585D386EB94 >/dev/null 2>&1 -add-apt-repository https://dl.hhvm.com/"${family,,}" >/dev/null 2>&1 -debconf-apt-progress -- apt-get update -install_packet "hhvm" "HipHop Virtual Machine" -} - - - - -install_phpmyadmin () -{ -# -# Phpmyadmin unattended installation -# -if [[ "$family" != "Ubuntu" ]]; then -DEBIAN_FRONTEND=noninteractive debconf-apt-progress -- apt-get -y install phpmyadmin -else -debconf-set-selections <<< "phpmyadmin phpmyadmin/internal/skip-preseed boolean true" -debconf-set-selections <<< "phpmyadmin phpmyadmin/reconfigure-webserver multiselect true" -debconf-set-selections <<< "phpmyadmin phpmyadmin/dbconfig-install boolean false" -echo "phpmyadmin phpmyadmin/internal/skip-preseed boolean true" | debconf-set-selections -echo "phpmyadmin phpmyadmin/reconfigure-webserver multiselect" | debconf-set-selections -echo "phpmyadmin phpmyadmin/dbconfig-install boolean false" | debconf-set-selections -debconf-apt-progress -- apt-get install -y phpmyadmin -fi -} - - - - -install_apache () -{ -# -# Install Apache2, PHP5, FCGI, suExec, Pear and mcrypt -# - -local pkg="apache2 apache2-doc apache2-utils libapache2-mod-fcgid php-pear mcrypt imagemagick libruby libapache2-mod-python memcached" - -local pkg_xenial="libapache2-mod-php php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi \ -apache2-suexec-pristine php-auth php7.0-mcrypt php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 php7.0-tidy \ -php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring php7.0-opcache php-apcu \ -libapache2-mod-fastcgi php7.0-fpm letsencrypt" - -local pkg_stretch="libapache2-mod-php php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi libapache2-mod-fcgid \ -apache2-suexec-pristine php7.0-mcrypt libapache2-mod-python php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 \ -php7.0-tidy php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring libapache2-mod-passenger \ -php7.0-soap php7.0-fpm php7.0-opcache php-apcu certbot" - -local pkg_jessie="apache2.2-common apache2-mpm-prefork libexpat1 ssl-cert libapache2-mod-php5 php5 php5-common php5-gd php5-mysql \ -php5-imap php5-cli php5-cgi libapache2-mod-fcgid apache2-suexec php-pear php-auth php5-mcrypt mcrypt php5-imagick libapache2-mod-python \ -php5-curl php5-intl php5-memcache php5-memcached php5-pspell php5-recode php5-sqlite php5-tidy php5-xmlrpc php5-xsl \ -libapache2-mod-passenger php5-xcache libapache2-mod-fastcgi php5-fpm" - -local temp="pkg_${distribution}" -install_packet "${pkg} ${!temp}" "Apache for $family $distribution" -# fix HTTPOXY vulnerability -cat < /etc/apache2/conf-available/httpoxy.conf - - RequestHeader unset Proxy early - - -EOT - -a2enmod actions proxy_fcgi setenvif fastcgi alias httpoxy suexec rewrite ssl actions include dav_fs dav auth_digest cgi headers >/dev/null 2>&1 -a2enconf php7.0-fpm >/dev/null 2>&1 -service apache2 restart >> /dev/null -} - - - - -install_nginx () -{ -# -# Install NginX, PHP5, FCGI, suExec, Pear, And mcrypt -# -local pkg="nginx php-pear memcached fcgiwrap" - -local pkg_xenial="php7.0-fpm php7.0-opcache php7.0-fpm php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi \ -php7.0-mcrypt mcrypt imagemagick libruby php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 php7.0-tidy \ -php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring php-apcu" - -local pkg_stretch="php7.0-fpm php7.0-opcache php7.0-fpm php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi \ -php7.0-mcrypt mcrypt imagemagick libruby php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 php7.0-tidy \ -php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring php-apcu" - -local pkg_jessie="php5-fpm php5-mysql php5-curl php5-gd php5-intl php5-imagick php5-imap php5-mcrypt php5-memcache \ -php5-memcached php5-pspell php5-recode php5-snmp php5-sqlite php5-tidy php5-xmlrpc php5-xsl php-apc" - -local temp="pkg_${distribution}" -install_packet "${pkg} ${!temp}" "Nginx for $family $distribution" - -phpenmod mcrypt mbstring - -if [[ -f /etc/php/7.0/fpm/php.ini ]]; then - tz=$(cat /etc/timezone | sed 's/\//\\\//g') - sed -i "s/^cgi.fix_pathinfo=.*/cgi.fix_pathinfo=0/" /etc/php/7.0/fpm/php.ini - sed -i "s/^date.timezone=.*/date.timezone=""$tz""/" /etc/php/7.0/fpm/php.ini - service php7.0-fpm reload >> /dev/null -else - debconf-apt-progress -- apt-get install -y python-certbot -t jessie-backports - service php5-fpm reload >> /dev/null -fi -} - - - - -install_PureFTPD () -{ -# -# Install PureFTPd and Quota -# -install_packet "pure-ftpd-common pure-ftpd-mysql quota quotatool" "pureFTPd and Quota" - -sed -i 's/VIRTUALCHROOT=false/VIRTUALCHROOT=true/' /etc/default/pure-ftpd-common -echo 1 > /etc/pure-ftpd/conf/TLS -mkdir -p /etc/ssl/private/ -openssl req -x509 -nodes -days 7300 -newkey rsa:2048 -subj "/C=GB/ST=GB/L=GB/O=GB/OU=GB/CN=$(hostname -f)/emailAddress=joe@joe.com" -keyout /etc/ssl/private/pure-ftpd.pem -out /etc/ssl/private/pure-ftpd.pem >/dev/null 2>&1 -chmod 600 /etc/ssl/private/pure-ftpd.pem -/etc/init.d/pure-ftpd-mysql restart >/dev/null 2>&1 -local temp=$(cat /etc/fstab | grep "/ " | tail -1 | awk '{print $4}') -sed -i "s/$temp/$temp,usrjquota=quota.user,grpjquota=quota.group,jqfmt=vfsv0/" /etc/fstab -mount -o remount / >/dev/null 2>&1 -quotacheck -avugm >/dev/null 2>&1 -quotaon -avug >/dev/null 2>&1 -} - - - - -install_Bind () -{ -# -# Install BIND DNS Server -# -install_packet "bind9 dnsutils" "Install BIND DNS Server" -} - - - - -install_Stats () -{ -# -# Install Vlogger, Webalizer, And AWstats -# -install_packet "vlogger webalizer awstats geoip-database libclass-dbi-mysql-perl" "vlogger, webalizer, awstats" -sed -i "s/*/10 * * * * www-data/#*/10 * * * * www-data/" /etc/cron.d/awstats -sed -i "s/10 03 * * * www-data/#10 03 * * * www-data/" /etc/cron.d/awstats -} - - - - -install_Jailkit() -{ -# -debconf-apt-progress -- apt-get install -y build-essential autoconf automake libtool flex bison debhelper binutils -cd ${TEMP_DIR} -wget -q http://olivier.sessink.nl/jailkit/jailkit-2.19.tar.gz -O - | tar -xz && cd jailkit-2.19 -echo 5 > debian/compat -./debian/rules binary > /dev/null 2>&1 -dpkg -i ../jailkit_2.19-1_*.deb > /dev/null 2>&1 -} - - - - -install_Fail2BanDovecot() -{ -# -# Install fail2ban -# -install_packet "fail2ban ufw" "Install fail2ban and UFW Firewall" -if [[ $distribution == "stretch" ]]; then -cat > /etc/fail2ban/jail.local <<"EOF" -[pure-ftpd] -enabled = true -port = ftp -filter = pure-ftpd -logpath = /var/log/syslog -maxretry = 3 - -[dovecot] -enabled = true -filter = dovecot -logpath = /var/log/mail.log -maxretry = 5 - -[postfix-sasl] -enabled = true -port = smtp -filter = postfix-sasl -logpath = /var/log/mail.log -maxretry = 3 -EOF -else -cat > /etc/fail2ban/jail.local <<"EOF" -[pureftpd] -enabled = true -port = ftp -filter = pureftpd -logpath = /var/log/syslog -maxretry = 3 - -[dovecot-pop3imap] -enabled = true -filter = dovecot-pop3imap -action = iptables-multiport[name=dovecot-pop3imap, port="pop3,pop3s,imap,imaps", protocol=tcp] -logpath = /var/log/mail.log -maxretry = 5 - -[sasl] -enabled = true -port = smtp -filter = postfix-sasl -logpath = /var/log/mail.log -maxretry = 3 -EOF -fi -} - - - - -install_Fail2BanRulesDovecot() -{ -# -# Dovecot rules -# -cat > /etc/fail2ban/filter.d/pureftpd.conf <<"EOF" -[Definition] -failregex = .*pure-ftpd: \(.*@\) \[WARNING\] Authentication failed for user.* -ignoreregex = -EOF - -cat > /etc/fail2ban/filter.d/dovecot-pop3imap.conf <<"EOF" -[Definition] -failregex = (?: pop3-login|imap-login): .*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed|Aborted login \(\d+ authentication attempts).*rip=(?P\S*),.* -ignoreregex = -EOF -# Add the missing ignoreregex line -echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf -service fail2ban restart >> /dev/null -} - - - - -install_ISPConfig (){ -#------------------------------------------------------------------------------------------------------------------------------------------ -# Install ISPConfig 3 -#------------------------------------------------------------------------------------------------------------------------------------------ -cd ${TEMP_DIR} -wget -q http://www.ispconfig.org/downloads/ISPConfig-3-stable.tar.gz -O - | tar -xz -cd ${TEMP_DIR}/ispconfig3_install/install/ -#apt-get -y install php5-cli php5-mysql -php -q install.php --autoinstall=${TEMP_DIR}/isp.conf.php -echo "Admin panel: https://$serverIP:8080" -echo "PHPmyadmin: http://$serverIP:8081/phpmyadmin" -} - - -install_mayan_edms (){ -# -# Install Mayan EDMS -# - -# Default values -MAYAN_DATABASE_PASSWORD="mayandbpass" -MAYAN_INSTALLATION_FOLDER="/opt/mayan-edms" -MAYAN_MEDIA_ROOT="/opt/mayan-edms-data" - -# User interaction -exec 3>&1 -dialog --title "Server configuration" --separate-widget $'\n' \ ---ok-label "Install" --backtitle "$BACKTITLE" \ ---form "\nPlease fill out this form:\n " 13 70 0 \ -"Ddatabase password:" 1 1 "$MAYAN_DATABASE_PASSWORD" 1 31 32 0 \ -"Installation folder:" 2 1 "$MAYAN_INSTALLATION_FOLDER" 2 31 32 0 \ -"Data folder:" 3 1 "$MAYAN_MEDIA_ROOT" 3 31 32 0 \ -2>&1 1>&3 | { -read -r MAYAN_DATABASE_PASSWORD -read -r MAYAN_MEDIA_ROOT -read -r MAYAN_INSTALLATION_FOLDER -echo $MAYAN_DATABASE_PASSWORD > ${TEMP_DIR}/MAYAN_DATABASE_PASSWORD -echo $MAYAN_MEDIA_ROOT > ${TEMP_DIR}/MAYAN_MEDIA_ROOT -echo $MAYAN_INSTALLATION_FOLDER > ${TEMP_DIR}/MAYAN_INSTALLATION_FOLDER -} -exec 3>&- -read MAYAN_DATABASE_PASSWORD < ${TEMP_DIR}/MAYAN_DATABASE_PASSWORD -read MAYAN_MEDIA_ROOT < ${TEMP_DIR}/MAYAN_MEDIA_ROOT -read MAYAN_INSTALLATION_FOLDER < ${TEMP_DIR}/MAYAN_INSTALLATION_FOLDER - -# OS dependencies -install_packet "g++ gcc ghostscript gnupg1 graphviz libffi-dev libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice libssl-dev libtiff-dev poppler-utils postgresql python-dev python-pip python-virtualenv redis-server sane-utils supervisor tesseract-ocr zlib1g-dev" "Installing dependencies" - -# Mayan OS user account -dialog --infobox "Adding Mayan EDMS user account" 3 70 -adduser mayan --disabled-password --disabled-login --no-create-home --gecos "" >/dev/null 2>&1 -sleep 1 - -# Create installtion and data folders -mkdir -p "${MAYAN_INSTALLATION_FOLDER}" -mkdir -p "${MAYAN_MEDIA_ROOT}" - -# Create the Python virtualenv to isolate Python dependencies of Mayan -dialog --infobox "Creating Python virtual environment" 3 70 -python /usr/lib/python2.7/dist-packages/virtualenv.py $MAYAN_INSTALLATION_FOLDER > /dev/null - -# Give ownership to the Mayan OS user -chown mayan:mayan "${MAYAN_INSTALLATION_FOLDER}" -R -chown mayan:mayan "${MAYAN_MEDIA_ROOT}" -R - -# Pillow can't find zlib or libjpeg on aarch64 (ODROID C2) -if [ "$(uname -m)" = "aarch64" ]; then \ - ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \ - ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \ -; fi - -# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1) -if [ "$(uname -m)" = "armv7l" ]; then \ - ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ && \ - ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \ -; fi - -# Install Mayan from the web and all its Python dependencies -MAYAN_PIP=$MAYAN_INSTALLATION_FOLDER/bin/pip -dialog --infobox "Installing Mayan EDMS Python package (Takes several minutes)" 3 70 -sudo -u mayan $MAYAN_PIP install --no-cache-dir mayan-edms > /dev/null 2>&1 - -# Python Postgres driver -dialog --infobox "Installing PostgreSQL database driver" 3 70 -sudo -u mayan $MAYAN_PIP install --no-cache-dir psycopg2==2.7.3.2 > /dev/null - -# Python Redis driver -dialog --infobox "Installing Redis driver" 3 70 -sudo -u mayan $MAYAN_PIP install --no-cache-dir redis==2.10.6 > /dev/null - -# Create postgres Mayan user and database -MAYAN_BIN=$MAYAN_INSTALLATION_FOLDER/bin/mayan-edms.py -dialog --infobox "Creating and initializing database (Takes several minutes)" 3 70 -sudo -u postgres psql -c "CREATE USER mayan WITH password '$MAYAN_DATABASE_PASSWORD';" -sudo -u postgres createdb -O mayan mayan - -# Execute initialsetup command. Migrate DB, create base files, downloads Javascript libraries -sudo -u mayan \ - MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \ - MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 \ - MAYAN_MEDIA_ROOT=$MAYAN_MEDIA_ROOT \ - MAYAN_DATABASE_PASSWORD=$MAYAN_DATABASE_PASSWORD \ - $MAYAN_BIN initialsetup --force > /dev/null - -# Compress and merge Javascript, CSS for web serving -dialog --infobox "Preparing static files" 3 70 -sudo -u mayan \ - MAYAN_MEDIA_ROOT=$MAYAN_MEDIA_ROOT \ - $MAYAN_BIN preparestatic --noinput > /dev/null - -# Create supervisor file for gunicorn (frontend), 3 background workers, and the scheduler for periodic tasks -cat > /etc/supervisor/conf.d/mayan.conf <> /etc/redis/redis.conf - -# This starts all of Mayan's processes -dialog --infobox "Starting service" 3 70 -systemctl restart supervisor.service - -# Installation report -dialog --msgbox "Installation complete.\nInstallation folder: $MAYAN_INSTALLATION_FOLDER\nData folder: $MAYAN_MEDIA_ROOT\nPort: 8000" 10 70 -} - - -#------------------------------------------------------------------------------------------------------------------------------------------ -# Main choices -#------------------------------------------------------------------------------------------------------------------------------------------ - -# check for root -# -if [[ $EUID != 0 ]]; then - dialog --title "Warning" --infobox "\nThis script requires root privileges.\n\nExiting ..." 7 41 - sleep 3 - exit -fi - -# nameserver backup -if [ -d /etc/resolvconf/resolv.conf.d ]; then - echo 'nameserver 8.8.8.8' > /etc/resolvconf/resolv.conf.d/head - resolvconf -u -fi - -# Create a safe temporary directory -TEMP_DIR=$(mktemp -d || exit 1) -chmod 700 ${TEMP_DIR} -trap "rm -rf \"${TEMP_DIR}\" ; exit 0" 0 1 2 3 15 - -# Install basic stuff, we have to wait for other apt tasks to finish -# (eg unattended-upgrades) -i=0 -tput sc -while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do - case $(($i % 4)) in - 0 ) j="-" ;; - 1 ) j="\\" ;; - 2 ) j="|" ;; - 3 ) j="/" ;; - esac - tput rc - echo -en "\r[$j] Waiting for other software managers to finish..." - sleep 0.5 - ((i=i+1)) -done - -apt-get -qq -y --no-install-recommends install debconf-utils html2text apt-transport-https dialog whiptail lsb-release bc expect > /dev/null - -# gather some info -# -TTY_X=$(($(stty size | awk '{print $2}')-6)) # determine terminal width -TTY_Y=$(($(stty size | awk '{print $1}')-6)) # determine terminal height -distribution=$(lsb_release -cs) -family=$(lsb_release -is) -serverIP=$(ip route get 8.8.8.8 | awk '{ print $NF; exit }') -set ${serverIP//./ } -SUBNET="$1.$2.$3." -hostnamefqdn=$(hostname -f) -mysql_pass="" -BACKTITLE="Softy - Armbian post deployment scripts, http://www.armbian.com" -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -#check_status - -# main dialog routine -# -DIALOG_CANCEL=1 -DIALOG_ESC=255 - -while true; do - - # prepare menu items - check_status - LISTLENGHT="$((${#LIST[@]}/2))" - exec 3>&1 - selection=$(dialog --backtitle "$BACKTITLE" --title "Installing to $family $distribution" --colors --clear --cancel-label \ - "Exit" --checklist "\nChoose what you want to install:\n " $(($LISTLENGHT+$LIST_CONST)) 70 15 "${LIST[@]}" 2>&1 1>&3) - exit_status=$? - exec 3>&- - case $exit_status in - $DIALOG_ESC | $DIALOG_CANCEL) - clear - exit 1 - ;; - esac - - # cycle trought all install options - i=0 - - while [ "$i" -lt "$LISTLENGHT" ]; do - - if [[ "$selection" == *Samba* && "$SAMBA_STATUS" != "on" ]]; then - install_samba - selection=${selection//Samba/} - fi - - if [[ "$selection" == *CUPS* && "$CUPS_STATUS" != "on" ]]; then - install_cups - selection=${selection//CUPS/} - fi - - if [[ "$selection" == *headend* && "$TVHEADEND_STATUS" != "on" ]]; then - install_tvheadend - selection=${selection//\"TV headend\"/} - fi - - if [[ "$selection" == *Minidlna* && "$MINIDLNA_STATUS" != "on" ]]; then - install_packet "minidlna" "Install lightweight DLNA/UPnP-AV server" - selection=${selection//Minidlna/} - fi - - if [[ "$selection" == *ISPConfig* && "$ISPCONFIG_STATUS" != "on" ]]; then - server_conf - if [[ "$MYSQL_PASS" == "" ]]; then - dialog --msgbox "Mysql password can't be blank. Exiting..." 7 70 - exit - fi - if [[ "$(echo $HOSTNAMEFQDN | grep -P '(?=^.{1,254}$)(^(?>(?!\d+\.)[a-zA-Z0-9_\-]{1,63}\.?)+(?:[a-zA-Z]{2,})$)')" == "" ]]; then - dialog --msgbox "Invalid FQDN. Exiting..." 7 70 - exit - fi - choose_webserver; install_basic; install_DashNTP; install_MySQL; install_MySQLDovecot; install_Virus; install_$server; - install_phpmyadmin - [[ -z "$(dpkg --print-architecture | grep arm)" ]] && install_hhvm - create_ispconfig_configuration;install_PureFTPD; - install_Jailkit; install_Fail2BanDovecot; install_Fail2BanRulesDovecot; - install_ISPConfig - read -n 1 -s -p "Press any key to continue" - selection=${selection//ISPConfig/} - fi - - if [[ "$selection" == *Syncthing* && "$SYNCTHING_STATUS" != "on" ]]; then - install_syncthing - selection=${selection//Syncthing/} - fi - - if [[ "$selection" == *ExaGear* && "$EXAGEAR_STATUS" != "on" ]]; then - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install exagear-armbian exagear-desktop exagear-dsound-server exagear-guest-ubuntu-1604 - selection=${selection//ExaGear/} - fi - - if [[ "$selection" == *server* && "$VPN_SERVER_STATUS" != "on" ]]; then - install_vpn_server - selection=${selection//\"VPN server\"/} - fi - - if [[ "$selection" == *client* && "$VPN_CLIENT_STATUS" != "on" ]]; then - install_vpn_client - selection=${selection//\"VPN client\"/} - fi - if [[ "$selection" == *NCP* && "$NCP_STATUS" != "on" ]]; then - install_ncp - selection=${selection//NCP/} - fi - - if [[ "$selection" == *OMV* && "$OMV_STATUS" != "on" ]]; then - install_omv - selection=${selection//OMV/} - fi - - if [[ "$selection" == *Plex* && "$PLEX_STATUS" != "on" ]]; then - install_plex_media_server - selection=${selection//Plex/} - fi - - if [[ "$selection" == *Radarr* && "$RADARR_STATUS" != "on" ]]; then - install_radarr - selection=${selection//Radarr/} - fi - - if [[ "$selection" == *Sonarr* && "$SONARR_STATUS" != "on" ]]; then - install_sonarr - selection=${selection//Sonarr/} - fi - - if [[ "$selection" == *hole* && "$PI_HOLE_STATUS" != "on" ]]; then - curl -L "https://install.pi-hole.net" | bash - selection=${selection//\"Pi hole\"/} - fi - - if [[ "$selection" == *Transmission* && "$TRANSMISSION_STATUS" != "on" ]]; then - install_transmission - selection=${selection//Transmission/} - dialog --title "Seed Armbian torrents" --backtitle "$BACKTITLE" --yes-label "Yes" --no-label "Cancel" --yesno "\ - \nDo you want to help community and seed armbian torrent files? It will ensure faster download for everyone.\ - \n\nWe need around 80Gb of your space." 11 44 - if [[ $? = 0 ]]; then - install_transmission_seed_armbian_torrents - fi - fi - - if [[ "$selection" == *UrBackup* && "$URBACKUP_STATUS" != "on" ]]; then - install_urbackup - selection=${selection//UrBackup/} - fi - - if [[ "$selection" == *Mayan* && "$MAYAN_STATUS" != "on" ]]; then - install_mayan_edms - selection=${selection//\"Mayan EDMS\"/} - fi - - i=$[$i+1] - done - # reread statuses - check_status -done - - - diff --git a/contrib/scripts/install/production.sh b/contrib/scripts/install/production.sh deleted file mode 100644 index 2f70812fec..0000000000 --- a/contrib/scripts/install/production.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash - -# ====== CONFIG ====== -INSTALLATION_DIRECTORY=/usr/share/mayan-edms/ -DB_NAME=mayan_edms -DB_USERNAME=mayan -DB_PASSWORD=test123 -# ==== END CONFIG ==== - -cat << EOF | tee -a /etc/motd.tail -********************************** - -Mayan EDMS Vagrant Production Box - -********************************** -EOF - -echo -e "\n -> Running apt-get update & upgrade \n" -apt-get -qq update -apt-get -y upgrade - -echo -e "\n -> Installing core binaries \n" -apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y - -echo -e "\n -> Setting up virtualenv \n" -rm -f ${INSTALLATION_DIRECTORY} -virtualenv ${INSTALLATION_DIRECTORY} -source ${INSTALLATION_DIRECTORY}bin/activate - -echo -e "\n -> Installing Mayan EDMS from PyPI \n" -pip install mayan-edms - -echo -e "\n -> Installing Python client for PostgreSQL, Redis, and uWSGI \n" -pip install psycopg2 redis uwsgi - -echo -e "\n -> Creating the database for the installation \n" -echo "CREATE USER mayan WITH PASSWORD '$DB_PASSWORD';" | sudo -u postgres psql -sudo -u postgres createdb -O $DB_USERNAME $DB_NAME - -echo -e "\n -> Creating the directories for the logs \n" -mkdir /var/log/mayan - -echo -e "\n -> Making a convenience symlink \n" -cd ${INSTALLATION_DIRECTORY} -ln -s lib/python2.7/site-packages/mayan . - -echo -e "\n -> Creating an initial settings file \n" -mayan-edms.py createsettings - -echo -e "\n -> Updating the mayan/settings/local.py file \n" -cat >> mayan/settings/local.py << EOF -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': '$DB_NAME', - 'USER': '$DB_USERNAME', - 'PASSWORD': '$DB_PASSWORD', - 'HOST': 'localhost', - 'PORT': '5432', - } -} - -BROKER_URL = 'redis://127.0.0.1:6379/0' -CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0' -EOF - -echo -e "\n -> Migrating the database or initialize the project \n" -mayan-edms.py initialsetup - -echo -e "\n -> Disabling the default NGINX site \n" -rm -f /etc/nginx/sites-enabled/default - -echo -e "\n -> Creating a uwsgi.ini file \n" -cat > uwsgi.ini << EOF -[uwsgi] -chdir = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan -chmod-socket = 664 -chown-socket = www-data:www-data -env = DJANGO_SETTINGS_MODULE=mayan.settings.production -gid = www-data -logto = /var/log/uwsgi/%n.log -pythonpath = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages -master = True -max-requests = 5000 -socket = ${INSTALLATION_DIRECTORY}uwsgi.sock -uid = www-data -vacuum = True -wsgi-file = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan/wsgi.py -EOF - -echo -e "\n -> Creating the directory for the uWSGI log files \n" -mkdir -p /var/log/uwsgi - -echo -e "\n -> Creating the NGINX site file for Mayan EDMS, /etc/nginx/sites-available/mayan \n" -cat > /etc/nginx/sites-available/mayan << EOF -server { - listen 80; - server_name localhost; - - location / { - include uwsgi_params; - uwsgi_pass unix:${INSTALLATION_DIRECTORY}uwsgi.sock; - - client_max_body_size 30M; # Increse if your plan to upload bigger documents - proxy_read_timeout 30s; # Increase if your document uploads take more than 30 seconds - } - - location /static { - alias ${INSTALLATION_DIRECTORY}mayan/media/static; - expires 1h; - } - - location /favicon.ico { - alias ${INSTALLATION_DIRECTORY}mayan/media/static/appearance/images/favicon.ico; - expires 1h; - } -} -EOF - -echo -e "\n -> Enabling the NGINX site for Mayan EDMS \n" -ln -s /etc/nginx/sites-available/mayan /etc/nginx/sites-enabled/ - -echo -e "\n -> Creating the supervisor file for the uWSGI process, /etc/supervisor/conf.d/mayan-uwsgi.conf \n" -cat > /etc/supervisor/conf.d/mayan-uwsgi.conf << EOF -[program:mayan-uwsgi] -command = ${INSTALLATION_DIRECTORY}bin/uwsgi --ini ${INSTALLATION_DIRECTORY}uwsgi.ini -user = root -autostart = true -autorestart = true -redirect_stderr = true -EOF - -echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n" -cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF -[program:mayan-worker] -command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -directory = ${INSTALLATION_DIRECTORY} -user = www-data -stdout_logfile = /var/log/mayan/worker-stdout.log -stderr_logfile = /var/log/mayan/worker-stderr.log -autostart = true -autorestart = true -startsecs = 10 -stopwaitsecs = 10 -killasgroup = true -priority = 998 - -[program:mayan-beat] -command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR -directory = ${INSTALLATION_DIRECTORY} -user = www-data -numprocs = 1 -stdout_logfile = /var/log/mayan/beat-stdout.log -stderr_logfile = /var/log/mayan/beat-stderr.log -autostart = true -autorestart = true -startsecs = 10 -stopwaitsecs = 1 -killasgroup = true -priority = 998 -EOF - -echo -e "\n -> Collecting the static files \n" -mayan-edms.py preparestatic --noinput - -echo -e "\n -> Making the installation directory readable and writable by the webserver user \n" -chown www-data:www-data ${INSTALLATION_DIRECTORY} -R - -echo -e "\n -> Restarting the services \n" -/etc/init.d/nginx restart -/etc/init.d/supervisor restart diff --git a/contrib/scripts/process_messages.py b/contrib/scripts/process_messages.py index f473f050b6..29b7efc416 100755 --- a/contrib/scripts/process_messages.py +++ b/contrib/scripts/process_messages.py @@ -14,7 +14,7 @@ APP_LIST = ( 'django_gpg', 'document_comments', 'document_indexing', 'document_parsing', 'document_signatures', 'document_states', 'documents', 'dynamic_search', 'events', 'file_metadata', 'linking', - 'lock_manager', 'mayan_statistics', 'mailer', 'metadata', 'mirroring', + 'lock_manager', 'mailer', 'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings', 'sources', 'storage', 'tags', 'task_manager', 'user_management' diff --git a/contrib/scripts/start_gunicorn.sh b/contrib/scripts/start_gunicorn.sh deleted file mode 100644 index 9e52a959c4..0000000000 --- a/contrib/scripts/start_gunicorn.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -NAME="mayan-edms" -DJANGODIR=/usr/share/mayan-edms -SOCKFILE=/var/tmp/filesystem.sock -USER=www-data -GROUP=www-data -NUM_WORKERS=3 -DJANGO_SETTINGS_MODULE=mayan.settings.production -DJANGO_WSGI_MODULE=mayan.wsgi -TIMEOUT=600 - -echo "Starting $NAME as `whoami`" - -# Activate the virtual environment -cd $DJANGODIR -source bin/activate -export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE -export PYTHONPATH=$DJANGODIR:$PYTHONPATH - -# Create the run directory if it doesn't exist -RUNDIR=$(dirname $SOCKFILE) -test -d $RUNDIR || mkdir -p $RUNDIR - -# Start your Django Unicorn -# Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon) -exec bin/gunicorn ${DJANGO_WSGI_MODULE}:application \ - --name $NAME \ - --workers $NUM_WORKERS \ - --user=$USER --group=$GROUP \ - --log-level=debug \ - --bind=unix:$SOCKFILE \ - --timeout=$TIMEOUT - - diff --git a/docker/Dockerfile b/docker/Dockerfile index 14ae97ed69..2f36498999 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -122,7 +122,7 @@ RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \ # Install the built Mayan EDMS package && pip install --no-cache-dir --no-use-pep517 dist/mayan* \ # Install the static content -&& mayan-edms.py installjavascript \ +&& mayan-edms.py installdependencies \ && MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}" diff --git a/docker/rootfs/version b/docker/rootfs/version index 5ae69bd5f0..34cde5690e 100755 --- a/docker/rootfs/version +++ b/docker/rootfs/version @@ -1 +1 @@ -3.2.5 +3.2.6 diff --git a/docs/chapters/deploying.rst b/docs/chapters/deploying.rst index 425f3274f9..d3016b98a3 100644 --- a/docs/chapters/deploying.rst +++ b/docs/chapters/deploying.rst @@ -127,9 +127,8 @@ For another setup that offers more performance and scalability refer to the :: - sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py initialsetup @@ -148,9 +147,8 @@ For another setup that offers more performance and scalability refer to the ------------------------------------------------------------------------ :: - sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf diff --git a/docs/releases/3.2.6.rst b/docs/releases/3.2.6.rst new file mode 100644 index 0000000000..0e897a24c9 --- /dev/null +++ b/docs/releases/3.2.6.rst @@ -0,0 +1,111 @@ +Version 3.2.6 +============= + +Released: July 10, 2019 + + +Changes +------- + +- Remove the smart settings app * import. Following MERC 0005. +- Encode settings YAML before hashing. Avoids unicode issues with Python 3. +- Fix document icon used in the workflow runtime links. +- Add trashed date time label. +- Fix thumbnail generation issue. GitLab issue #637. + Thanks to Giacomo Cariello (@giacomocariello) for the report + and the merge request fixing the issue. + +Removals +-------- + +- None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt + +Type in the console:: + + sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.6 + +the requirements will also be updated automatically. + + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +^^^^^^^^^^^^ + +Perform these steps after updating the code from either step above. + +Make a backup of your supervisord file:: + + sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck + +Update the supervisord configuration file. Replace the environment +variables values show here with your respective settings. This step will refresh +the supervisord configuration file with the new queues and the latest +recommended layout:: + + sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf + +Edit the supervisord configuration file and update any setting the template +generator missed:: + + sudo vi /etc/supervisor/conf.d/mayan.conf + +Migrate existing database schema with:: + + sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py performupgrade + +Add new static media:: + + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +- None + + +Bugs fixed or issues closed +--------------------------- + +- :gitlab-issue:`637` Thumbnail generation bug + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 6a66ffeb19..6bc9685c3c 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -22,11 +22,55 @@ Changes - Add support for source column exclusion. - Backport workflow context support. - Backport workflow transitions field support. +- Backport workflow email action. +- Backport individual index rebuild support. +- Rename the installjavascript command to installdependencies. +- Remove database conversion command. +- Remove support for quoted configuration entries. Support unquoted, + nested dictionaries in the configuration. Requires manual + update of existing config.yml files. +- Support user specified locations for the configuration file with the + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and + CONFIGURATION_LAST_GOOD_FILEPATH + (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. +- Move bootstrapped settings code to their own module in the smart_settings apps. +- Remove individual database configuration options. All database configuration + is now done using MAYAN_DATABASES to mirror Django way of doing database setup. +- Added support for YAML encoded environment variables to the platform + templates apps. +- Move YAML code to its own module. Code now resides in common.serialization + in the form of two new functions: yaml_load and yaml_dump. +- Move Django and Celery settings. Django settings now reside in the smart + settings app. Celery settings now reside in the task manager app. +- Backport FakeStorageSubclass from versions/next. Placeholder class to allow + serializing the real storage subclass to support migrations. + Used by all configurable storages. +- Support checking in and out multiple documents. +- Remove encapsulate helper. +- Add support for menu inheritance. +- Emphasize source column labels. Removals -------- -- None +- Database conversion. Reason for removal. The database conversions support + provided by this feature (SQLite to PostgreSQL) was being confused with + database migrations and upgrades. + + Database upgrades are the responsibility of the app and the framework. + Database conversions however are not the responsibility of the app (Mayan), + they are the responsibility of the framework. + + Database conversion is outside the scope of what Mayan does but we added + the code, management command, instructions and testing setup to provide + this to our users until the framework (Django) decided to add this + themselves (like they did with migrations). + + Continued confusion about the purpose of the feature and confusion about + how errors with this feature were a reflexion of the code quality of + Mayannecessitated the removal of the database conversion feature. + +- Django environ Upgrading from a previous version @@ -37,11 +81,11 @@ If installed via Python's PIP Remove deprecated requirements:: - $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt Type in the console:: - $ pip install mayan-edms==3.3 + /opt/mayan-edms/bin/pip install mayan-edms==3.3 the requirements will also be updated automatically. @@ -51,19 +95,19 @@ Using Git If you installed Mayan EDMS by cloning the Git repository issue the commands:: - $ git reset --hard HEAD - $ git pull + git reset --hard HEAD + git pull otherwise download the compressed archived and uncompress it overriding the existing installation. Remove deprecated requirements:: - $ pip uninstall -y -r removals.txt + pip uninstall -y -r removals.txt Next upgrade/add the new requirements:: - $ pip install --upgrade -r requirements.txt + pip install --upgrade -r requirements.txt Common steps @@ -80,9 +124,8 @@ variables values show here with your respective settings. This step will refresh the supervisord configuration file with the new queues and the latest recommended layout:: - sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf Edit the supervisord configuration file and update any setting the template @@ -92,11 +135,11 @@ generator missed:: Migrate existing database schema with:: - $ mayan-edms.py performupgrade + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade Add new static media:: - $ mayan-edms.py preparestatic --noinput + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput The upgrade procedure is now complete. @@ -104,12 +147,20 @@ The upgrade procedure is now complete. Backward incompatible changes ----------------------------- -- None +- Update quoted settings to be unquoted: + + - COMMON_SHARED_STORAGE_ARGUMENTS + - CONVERTER_GRAPHICS_BACKEND_ARGUMENTS + - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS + - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS + - FILE_METADATA_DRIVERS_ARGUMENTS + - SIGNATURES_STORAGE_BACKEND_ARGUMENTS Bugs fixed or issues closed --------------------------- - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified +- :gitlab-issue:`634` Failing docker entrypoint when using secret config .. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 33d838bd3d..ad5f283a27 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -21,6 +21,7 @@ versions of the documentation contain the release notes for any later releases. :maxdepth: 1 3.3 + 3.2.6 3.2.5 3.2.4 3.2.3 diff --git a/mayan/__init__.py b/mayan/__init__.py index 3c4a65b10d..5c740e43d9 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' -__version__ = '3.2.5' -__build__ = 0x030205 -__build_string__ = 'v3.2.5_Fri Jul 5 16:39:17 2019 -0400' +__version__ = '3.2.6' +__build__ = 0x030206 +__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' diff --git a/mayan/apps/appearance/settings.py b/mayan/apps/appearance/settings.py index 5b93d43525..ad0927530a 100644 --- a/mayan/apps/appearance/settings.py +++ b/mayan/apps/appearance/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_MAXIMUM_TITLE_LENGTH diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 818e81c133..0cb7944b43 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -216,6 +216,10 @@ a i { font-weight: bold; } +.source-column-label { + font-weight: bold; +} + /* Content */ @media (min-width:1200px) { .container-fluid { @@ -264,8 +268,8 @@ a i { #ajax-spinner { position: fixed; - top: 12px; - right: 10px; + top: 16px; + left: 10px; z-index: 9999; width: 25px; height: 25px; diff --git a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html index 991162c0c0..e0c4166cef 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html @@ -86,7 +86,7 @@ {% if not hide_columns %} {% navigation_get_source_columns source=object exclude_identifier=True as source_columns %} {% for column in source_columns %} -
{% navigation_source_column_resolve column=column as column_value %}{% if column_value != '' %}{% if column.include_label %}{{ column.label }}: {% endif %}{{ column_value }}{% endif %}
+
{% navigation_source_column_resolve column=column as column_value %}{% if column_value != '' %}{% if column.include_label %}{{ column.label }}: {% endif %}{{ column_value }}{% endif %}
{% endfor %} {% endif %} diff --git a/mayan/apps/authentication/settings.py b/mayan/apps/authentication/settings.py index de693442d5..ec264c7933 100644 --- a/mayan/apps/authentication/settings.py +++ b/mayan/apps/authentication/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_LOGIN_METHOD, DEFAULT_MAXIMUM_SESSION_LENGTH diff --git a/mayan/apps/autoadmin/settings.py b/mayan/apps/autoadmin/settings.py index d74b9b0d74..3c21d27339 100644 --- a/mayan/apps/autoadmin/settings.py +++ b/mayan/apps/autoadmin/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EMAIL, DEFAULT_PASSWORD, DEFAULT_USERNAME diff --git a/mayan/apps/checkouts/apps.py b/mayan/apps/checkouts/apps.py index 6f9e5b7c00..7c067edb0b 100644 --- a/mayan/apps/checkouts/apps.py +++ b/mayan/apps/checkouts/apps.py @@ -6,9 +6,12 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.acls.classes import ModelPermission from mayan.apps.common.apps import MayanAppConfig -from mayan.apps.common.menus import menu_facet, menu_main, menu_secondary +from mayan.apps.common.menus import ( + menu_facet, menu_main, menu_multi_item, menu_secondary +) from mayan.apps.dashboards.dashboards import dashboard_main from mayan.apps.events.classes import ModelEventType +from mayan.apps.navigation.classes import SourceColumn from .dashboard_widgets import DashboardWidgetTotalCheckouts from .events import ( @@ -17,8 +20,9 @@ from .events import ( ) from .handlers import handler_check_new_version_creation from .links import ( - link_check_in_document, link_check_out_document, link_check_out_info, - link_check_out_list + link_check_in_document, link_check_in_document_multiple, + link_check_out_document, link_check_out_document_multiple, + link_check_out_info, link_check_out_list ) from .methods import ( method_check_in, method_get_check_out_info, method_get_check_out_state, @@ -43,6 +47,8 @@ class CheckoutsApp(MayanAppConfig): def ready(self): super(CheckoutsApp, self).ready() + CheckedOutDocument = self.get_model(model_name='CheckedOutDocument') + DocumentCheckout = self.get_model(model_name='DocumentCheckout') Document = apps.get_model( app_label='documents', model_name='Document' ) @@ -76,6 +82,22 @@ class CheckoutsApp(MayanAppConfig): permission_document_check_out_detail_view ) ) + ModelPermission.register_inheritance( + model=DocumentCheckout, related='document' + ) + + SourceColumn( + attribute='get_user_display', include_label=True, order=99, + source=CheckedOutDocument + ) + SourceColumn( + attribute='get_checkout_datetime', include_label=True, order=99, + source=CheckedOutDocument + ) + SourceColumn( + attribute='get_checkout_expiration', include_label=True, order=99, + source=CheckedOutDocument + ) dashboard_main.add_widget( widget=DashboardWidgetTotalCheckouts, order=-1 @@ -85,6 +107,22 @@ class CheckoutsApp(MayanAppConfig): links=(link_check_out_info,), sources=(Document,) ) menu_main.bind_links(links=(link_check_out_list,), position=98) + menu_multi_item.bind_links( + links=( + link_check_in_document_multiple, + ), sources=(CheckedOutDocument,) + ) + menu_multi_item.bind_links( + links=( + link_check_in_document_multiple, + link_check_out_document_multiple, + ), sources=(Document,) + ) + menu_multi_item.unbind_links( + links=( + link_check_out_document_multiple, + ), sources=(CheckedOutDocument,) + ) menu_secondary.bind_links( links=(link_check_out_document, link_check_in_document), sources=( diff --git a/mayan/apps/checkouts/links.py b/mayan/apps/checkouts/links.py index d03d54c23d..c8012653c9 100644 --- a/mayan/apps/checkouts/links.py +++ b/mayan/apps/checkouts/links.py @@ -38,16 +38,26 @@ link_check_out_document = Link( args='object.pk', condition=is_not_checked_out, icon_class=icon_check_out_document, permissions=(permission_document_check_out,), - text=_('Check out document'), view='checkouts:check_out_document', + text=_('Check out document'), view='checkouts:check_out_document' +) +link_check_out_document_multiple = Link( + icon_class=icon_check_out_document, + permissions=(permission_document_check_out,), text=_('Check out'), + view='checkouts:check_out_document_multiple' ) link_check_in_document = Link( args='object.pk', icon_class=icon_check_in_document, condition=is_checked_out, permissions=( permission_document_check_in, permission_document_check_in_override - ), text=_('Check in document'), view='checkouts:check_in_document', + ), text=_('Check in document'), view='checkouts:check_in_document' +) +link_check_in_document_multiple = Link( + icon_class=icon_check_in_document, + permissions=(permission_document_check_in,), text=_('Check in'), + view='checkouts:check_in_document_multiple' ) link_check_out_info = Link( args='resolved_object.pk', icon_class=icon_check_out_info, permissions=( permission_document_check_out_detail_view, - ), text=_('Check in/out'), view='checkouts:check_out_info', + ), text=_('Check in/out'), view='checkouts:check_out_info' ) diff --git a/mayan/apps/checkouts/managers.py b/mayan/apps/checkouts/managers.py index 674a77257e..7b6b77635d 100644 --- a/mayan/apps/checkouts/managers.py +++ b/mayan/apps/checkouts/managers.py @@ -6,6 +6,7 @@ from django.apps import apps from django.db import models, transaction from django.utils.timezone import now +from mayan.apps.acls.models import AccessControlList from mayan.apps.documents.models import Document from .events import ( @@ -14,10 +15,53 @@ from .events import ( ) from .exceptions import DocumentNotCheckedOut from .literals import STATE_CHECKED_OUT, STATE_CHECKED_IN +from .permissions import ( + permission_document_check_in, permission_document_check_in_override +) logger = logging.getLogger(__name__) +class DocumentCheckoutBusinessLogicManager(models.Manager): + def check_in_document(self, document, user=None): + queryset = document._meta.default_manager.filter(pk=document.pk) + return self.check_in_documents(queryset=queryset, user=user) + + def check_in_documents(self, queryset, user=None): + if user: + user_document_checkouts = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in, + queryset=self.filter(user_id=user.pk, document__in=queryset), + user=user + ) + + others_document_checkouts = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in_override, + queryset=self.exclude(user_id=user.pk, document__in=queryset), + user=user + ) + + with transaction.atomic(): + if user: + for checkout in user_document_checkouts: + event_document_check_in.commit( + actor=user, target=checkout.document + ) + checkout.delete() + + for checkout in others_document_checkouts: + event_document_forceful_check_in.commit( + actor=user, target=checkout.document + ) + checkout.delete() + else: + for checkout in self.filter(document__in=queryset): + event_document_auto_check_in.commit( + target=checkout.document + ) + checkout.delete() + + class DocumentCheckoutManager(models.Manager): def are_document_new_versions_allowed(self, document, user=None): try: @@ -27,25 +71,6 @@ class DocumentCheckoutManager(models.Manager): else: return not check_out_info.block_new_version - def check_in_document(self, document, user=None): - try: - document_check_out = self.model.objects.get(document=document) - except self.model.DoesNotExist: - raise DocumentNotCheckedOut - else: - with transaction.atomic(): - if user: - if self.get_check_out_info(document=document).user != user: - event_document_forceful_check_in.commit( - actor=user, target=document - ) - else: - event_document_check_in.commit(actor=user, target=document) - else: - event_document_auto_check_in.commit(target=document) - - document_check_out.delete() - def check_in_expired_check_outs(self): for document in self.expired_check_outs(): document.check_in() @@ -57,7 +82,11 @@ class DocumentCheckoutManager(models.Manager): ) def checked_out_documents(self): - return Document.objects.filter( + CheckedOutDocument = apps.get_model( + app_label='checkouts', model_name='CheckedOutDocument' + ) + + return CheckedOutDocument.objects.filter( pk__in=self.model.objects.values('document__id') ) @@ -74,7 +103,11 @@ class DocumentCheckoutManager(models.Manager): return STATE_CHECKED_IN def expired_check_outs(self): - expired_list = Document.objects.filter( + CheckedOutDocument = apps.get_model( + app_label='checkouts', model_name='CheckedOutDocument' + ) + + expired_list = CheckedOutDocument.objects.filter( pk__in=self.model.objects.filter( expiration_datetime__lte=now() ).values_list('document__pk', flat=True) @@ -83,9 +116,6 @@ class DocumentCheckoutManager(models.Manager): return expired_list def get_by_natural_key(self, document_natural_key): - Document = apps.get_model( - app_label='documents', model_name='Document' - ) try: document = Document.objects.get_by_natural_key(document_natural_key) except Document.DoesNotExist: diff --git a/mayan/apps/checkouts/methods.py b/mayan/apps/checkouts/methods.py index 680f6cf4c7..3972807a8f 100644 --- a/mayan/apps/checkouts/methods.py +++ b/mayan/apps/checkouts/methods.py @@ -8,7 +8,7 @@ def method_check_in(self, user=None): app_label='checkouts', model_name='DocumentCheckout' ) - return DocumentCheckout.objects.check_in_document( + return DocumentCheckout.business_logic.check_in_document( document=self, user=user ) diff --git a/mayan/apps/checkouts/models.py b/mayan/apps/checkouts/models.py index c724e21db8..5efc322dff 100644 --- a/mayan/apps/checkouts/models.py +++ b/mayan/apps/checkouts/models.py @@ -14,7 +14,10 @@ from mayan.apps.documents.models import Document from .events import event_document_check_out from .exceptions import DocumentAlreadyCheckedOut -from .managers import DocumentCheckoutManager, NewVersionBlockManager +from .managers import ( + DocumentCheckoutBusinessLogicManager, DocumentCheckoutManager, + NewVersionBlockManager +) logger = logging.getLogger(__name__) @@ -49,6 +52,7 @@ class DocumentCheckout(models.Model): ) objects = DocumentCheckoutManager() + business_logic = DocumentCheckoutBusinessLogicManager() class Meta: ordering = ('pk',) @@ -81,13 +85,13 @@ class DocumentCheckout(models.Model): natural_key.dependencies = ['documents.Document'] def save(self, *args, **kwargs): - new_checkout = not self.pk - if not new_checkout or self.document.is_checked_out(): + is_new = not self.pk + if not is_new or self.document.is_checked_out(): raise DocumentAlreadyCheckedOut with transaction.atomic(): result = super(DocumentCheckout, self).save(*args, **kwargs) - if new_checkout: + if is_new: event_document_check_out.commit( actor=self.user, target=self.document ) @@ -119,3 +123,24 @@ class NewVersionBlock(models.Model): def natural_key(self): return self.document.natural_key() natural_key.dependencies = ['documents.Document'] + + +class CheckedOutDocument(Document): + class Meta: + proxy = True + + def get_user_display(self): + check_out_info = self.get_check_out_info() + return check_out_info.user.get_full_name() or check_out_info.user + + get_user_display.short_description = _('User') + + def get_checkout_datetime(self): + return self.get_check_out_info().checkout_datetime + + get_checkout_datetime.short_description = _('Checkout time and date') + + def get_checkout_expiration(self): + return self.get_check_out_info().expiration_datetime + + get_checkout_expiration.short_description = _('Checkout expiration') diff --git a/mayan/apps/checkouts/tests/mixins.py b/mayan/apps/checkouts/tests/mixins.py index fa8d601a4b..840fee34bf 100644 --- a/mayan/apps/checkouts/tests/mixins.py +++ b/mayan/apps/checkouts/tests/mixins.py @@ -4,13 +4,19 @@ import datetime from django.utils.timezone import now +from mayan.apps.common.literals import TIME_DELTA_UNIT_DAYS +from mayan.apps.common.tests.utils import as_id_list + from ..models import DocumentCheckout class DocumentCheckoutTestMixin(object): _test_document_check_out_seconds = 0.1 - def _check_out_test_document(self, user=None): + def _check_out_test_document(self, document=None, user=None): + if not document: + document = self.test_document + if not user: user = self._test_case_user @@ -19,7 +25,61 @@ class DocumentCheckoutTestMixin(object): ) self.test_check_out = DocumentCheckout.objects.check_out_document( - block_new_version=True, document=self.test_document, + block_new_version=True, document=document, expiration_datetime=self._check_out_expiration_datetime, user=user ) + + +class DocumentCheckoutViewTestMixin(object): + def _request_test_document_check_in_get_view(self): + return self.get( + viewname='checkouts:check_in_document', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_test_document_check_in_post_view(self): + return self.post( + viewname='checkouts:check_in_document', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_test_document_multiple_check_in_post_view(self): + return self.post( + viewname='checkouts:check_in_document_multiple', data={ + 'id_list': as_id_list(items=self.test_documents) + } + ) + + def _request_test_document_check_out_view(self): + return self.post( + viewname='checkouts:check_out_document', kwargs={ + 'pk': self.test_document.pk + }, data={ + 'block_new_version': True, + 'expiration_datetime_0': TIME_DELTA_UNIT_DAYS, + 'expiration_datetime_1': 2 + } + ) + + def _request_test_document_multiple_check_out_post_view(self): + return self.post( + viewname='checkouts:check_out_document_multiple', data={ + 'block_new_version': True, + 'expiration_datetime_0': TIME_DELTA_UNIT_DAYS, + 'expiration_datetime_1': 2, + 'id_list': as_id_list(items=self.test_documents) + } + ) + + def _request_test_document_check_out_detail_view(self): + return self.get( + viewname='checkouts:check_out_info', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_test_document_check_out_list_view(self): + return self.get(viewname='checkouts:check_out_list') diff --git a/mayan/apps/checkouts/tests/test_api.py b/mayan/apps/checkouts/tests/test_api.py index 71ec5ea536..d87bae3c17 100644 --- a/mayan/apps/checkouts/tests/test_api.py +++ b/mayan/apps/checkouts/tests/test_api.py @@ -65,7 +65,7 @@ class CheckoutsAPITestCase(DocumentCheckoutTestMixin, DocumentTestMixin, BaseAPI force_text(self.test_document.uuid) ) - def _request_document_checkout_view(self): + def _request_test_document_check_out_view(self): return self.post( viewname='rest_api:checkout-document-list', data={ 'document_pk': self.test_document.pk, @@ -74,7 +74,7 @@ class CheckoutsAPITestCase(DocumentCheckoutTestMixin, DocumentTestMixin, BaseAPI ) def test_document_checkout_no_access(self): - response = self._request_document_checkout_view() + response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual(DocumentCheckout.objects.count(), 0) @@ -82,7 +82,7 @@ class CheckoutsAPITestCase(DocumentCheckoutTestMixin, DocumentTestMixin, BaseAPI def test_document_checkout_with_access(self): self.grant_access(permission=permission_document_check_out, obj=self.test_document) - response = self._request_document_checkout_view() + response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual( diff --git a/mayan/apps/checkouts/tests/test_models.py b/mayan/apps/checkouts/tests/test_models.py index cd2ea0202f..dbfdd64d50 100644 --- a/mayan/apps/checkouts/tests/test_models.py +++ b/mayan/apps/checkouts/tests/test_models.py @@ -7,8 +7,7 @@ from mayan.apps.documents.tests import GenericDocumentTestCase, DocumentTestMixi from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH from ..exceptions import ( - DocumentAlreadyCheckedOut, DocumentNotCheckedOut, - NewDocumentVersionNotAllowed + DocumentAlreadyCheckedOut, NewDocumentVersionNotAllowed ) from ..models import DocumentCheckout, NewVersionBlock @@ -49,10 +48,6 @@ class DocumentCheckoutTestCase(DocumentCheckoutTestMixin, GenericDocumentTestCas block_new_version=True ) - def test_checkin_without_checkout(self): - with self.assertRaises(DocumentNotCheckedOut): - self.test_document.check_in() - def test_auto_check_in(self): self._check_out_test_document() diff --git a/mayan/apps/checkouts/tests/test_views.py b/mayan/apps/checkouts/tests/test_views.py index 6b4648145f..3910dedd98 100644 --- a/mayan/apps/checkouts/tests/test_views.py +++ b/mayan/apps/checkouts/tests/test_views.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from mayan.apps.common.literals import TIME_DELTA_UNIT_DAYS from mayan.apps.documents.permissions import permission_document_view from mayan.apps.documents.tests import GenericDocumentViewTestCase from mayan.apps.sources.links import link_document_version_upload @@ -12,64 +11,53 @@ from ..permissions import ( permission_document_check_out, permission_document_check_out_detail_view ) -from .mixins import DocumentCheckoutTestMixin +from .mixins import DocumentCheckoutTestMixin, DocumentCheckoutViewTestMixin -class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentViewTestCase): - def _request_document_check_in_get_view(self): - return self.get( - viewname='checkouts:check_in_document', kwargs={ - 'pk': self.test_document.pk - } - ) - - def test_check_in_document_get_view_no_permission(self): +class DocumentCheckoutViewTestCase( + DocumentCheckoutTestMixin, DocumentCheckoutViewTestMixin, + GenericDocumentViewTestCase +): + def test_document_check_in_get_view_no_permission(self): self._check_out_test_document() - response = self._request_document_check_in_get_view() - self.assertContains( - response=response, text=self.test_document.label, status_code=200 + response = self._request_test_document_check_in_get_view() + self.assertNotContains( + response=response, text=self.test_document.label, status_code=404 ) self.assertTrue(self.test_document.is_checked_out()) - def test_check_in_document_get_view_with_access(self): + def test_document_check_in_get_view_with_access(self): self._check_out_test_document() self.grant_access( obj=self.test_document, permission=permission_document_check_in ) - response = self._request_document_check_in_get_view() + response = self._request_test_document_check_in_get_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) self.assertTrue(self.test_document.is_checked_out()) - def _request_document_check_in_post_view(self): - return self.post( - viewname='checkouts:check_in_document', kwargs={ - 'pk': self.test_document.pk - } - ) - - def test_check_in_document_post_view_no_permission(self): + def test_document_check_in_post_view_no_permission(self): self._check_out_test_document() - response = self._request_document_check_in_post_view() - self.assertEqual(response.status_code, 403) + response = self._request_test_document_check_in_post_view() + self.assertEqual(response.status_code, 404) self.assertTrue(self.test_document.is_checked_out()) - def test_check_in_document_post_view_with_access(self): + def test_document_check_in_post_view_with_access(self): self._check_out_test_document() self.grant_access( obj=self.test_document, permission=permission_document_check_in ) - response = self._request_document_check_in_post_view() + response = self._request_test_document_check_in_post_view() self.assertEqual(response.status_code, 302) self.assertFalse(self.test_document.is_checked_out()) @@ -79,24 +67,93 @@ class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentVie ) ) - def _request_document_checkout_view(self): - return self.post( - viewname='checkouts:check_out_document', kwargs={ - 'pk': self.test_document.pk - }, data={ - 'expiration_datetime_0': 2, - 'expiration_datetime_1': TIME_DELTA_UNIT_DAYS, - 'block_new_version': True - } + def test_document_multiple_check_in_post_view_no_permission(self): + # Upload second document + self.upload_document() + + self._check_out_test_document(document=self.test_documents[0]) + self._check_out_test_document(document=self.test_documents[1]) + + response = self._request_test_document_multiple_check_in_post_view() + self.assertEqual(response.status_code, 404) + + self.assertTrue(self.test_documents[0].is_checked_out()) + self.assertTrue(self.test_documents[1].is_checked_out()) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) ) - def test_check_out_document_view_no_permission(self): - response = self._request_document_checkout_view() - self.assertEqual(response.status_code, 403) + def test_document_multiple_check_in_post_view_with_document_0_access(self): + # Upload second document + self.upload_document() + + self._check_out_test_document(document=self.test_documents[0]) + self._check_out_test_document(document=self.test_documents[1]) + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_in + ) + + response = self._request_test_document_multiple_check_in_post_view() + self.assertEqual(response.status_code, 302) + + self.assertFalse(self.test_documents[0].is_checked_out()) + self.assertTrue(self.test_documents[1].is_checked_out()) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_in_post_view_with_access(self): + # Upload second document + self.upload_document() + + self._check_out_test_document(document=self.test_documents[0]) + self._check_out_test_document(document=self.test_documents[1]) + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_in + ) + self.grant_access( + obj=self.test_documents[1], permission=permission_document_check_in + ) + + response = self._request_test_document_multiple_check_in_post_view() + self.assertEqual(response.status_code, 302) + + self.assertFalse(self.test_documents[0].is_checked_out()) + self.assertFalse(self.test_documents[1].is_checked_out()) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_check_out_view_no_permission(self): + response = self._request_test_document_check_out_view() + self.assertEqual(response.status_code, 404) self.assertFalse(self.test_document.is_checked_out()) - def test_check_out_document_view_with_access(self): + def test_document_check_out_view_with_access(self): self.grant_access( obj=self.test_document, permission=permission_document_check_out ) @@ -105,28 +162,117 @@ class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentVie permission=permission_document_check_out_detail_view ) - response = self._request_document_checkout_view() + response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, 302) self.assertTrue(self.test_document.is_checked_out()) - def _request_check_out_detail_view(self): - return self.get( - viewname='checkouts:check_out_info', kwargs={ - 'pk': self.test_document.pk - } + def test_document_multiple_check_out_post_view_no_permission(self): + # Upload second document + self.upload_document() + + self.grant_access( + obj=self.test_documents[0], + permission=permission_document_check_out_detail_view + ) + self.grant_access( + obj=self.test_documents[1], + permission=permission_document_check_out_detail_view ) - def test_checkout_detail_view_no_permission(self): + response = self._request_test_document_multiple_check_out_post_view() + self.assertEqual(response.status_code, 404) + + self.assertFalse(self.test_documents[0].is_checked_out()) + self.assertFalse(self.test_documents[1].is_checked_out()) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_out_post_view_with_document_0_access(self): + # Upload second document + self.upload_document() + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_out + ) + self.grant_access( + obj=self.test_documents[0], + permission=permission_document_check_out_detail_view + ) + self.grant_access( + obj=self.test_documents[1], + permission=permission_document_check_out_detail_view + ) + + response = self._request_test_document_multiple_check_out_post_view() + self.assertEqual(response.status_code, 302) + + self.assertTrue(self.test_documents[0].is_checked_out()) + self.assertFalse(self.test_documents[1].is_checked_out()) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_out_post_view_with_access(self): + # Upload second document + self.upload_document() + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_out + ) + self.grant_access( + obj=self.test_documents[1], permission=permission_document_check_out + ) + self.grant_access( + obj=self.test_documents[0], + permission=permission_document_check_out_detail_view + ) + self.grant_access( + obj=self.test_documents[1], + permission=permission_document_check_out_detail_view + ) + + response = self._request_test_document_multiple_check_out_post_view() + self.assertEqual(response.status_code, 302) + + self.assertTrue(self.test_documents[0].is_checked_out()) + self.assertTrue(self.test_documents[1].is_checked_out()) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_check_out_detail_view_no_permission(self): self._check_out_test_document() - response = self._request_check_out_detail_view() + response = self._request_test_document_check_out_detail_view() self.assertNotContains( response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=404 ) - def test_checkout_detail_view_with_access(self): + def test_document_check_out_detail_view_with_access(self): self._check_out_test_document() self.grant_access( @@ -134,15 +280,12 @@ class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentVie permission=permission_document_check_out_detail_view ) - response = self._request_check_out_detail_view() + response = self._request_test_document_check_out_detail_view() self.assertContains( response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=200 ) - def _request_check_out_list_view(self): - return self.get(viewname='checkouts:check_out_list') - - def test_checkout_list_view_no_permission(self): + def test_document_checkout_list_view_no_permission(self): self._check_out_test_document() self.grant_access( @@ -150,12 +293,12 @@ class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentVie permission=permission_document_view ) - response = self._request_check_out_list_view() + response = self._request_test_document_check_out_list_view() self.assertNotContains( response=response, text=self.test_document.label, status_code=200 ) - def test_checkout_list_view_with_access(self): + def test_document_checkout_list_view_with_access(self): self._check_out_test_document() self.grant_access( @@ -167,12 +310,12 @@ class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentVie permission=permission_document_view ) - response = self._request_check_out_list_view() + response = self._request_test_document_check_out_list_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) - def test_document_new_version_after_check_out(self): + def test_document_check_out_new_version(self): """ Gitlab issue #231 User shown option to upload new version of a document even though it @@ -209,45 +352,39 @@ class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentVie self.assertEqual(resolved_link, None) - def test_forcefull_check_in_document_view_no_permission(self): + def test_document_check_in_forcefull_view_no_permission(self): # Gitlab issue #237 # Forcefully checking in a document by a user without adequate # permissions throws out an error - self._create_test_case_superuser() - self._check_out_test_document(user=self._test_case_superuser) + self._create_test_user() + self._check_out_test_document(user=self.test_user) self.grant_access( obj=self.test_document, permission=permission_document_check_in ) - response = self.post( - viewname='checkouts:check_in_document', kwargs={ - 'pk': self.test_document.pk - } - ) - self.assertContains( - response=response, text='Insufficient permissions', status_code=403 - ) - - self.assertTrue(self.test_document.is_checked_out()) - - def test_forcefull_check_in_document_view_with_permission(self): - self._create_test_case_superuser() - self._check_out_test_document(user=self._test_case_superuser) - - self.grant_access( - obj=self.test_document, permission=permission_document_check_in - ) - self.grant_access( - obj=self.test_document, permission=permission_document_check_in_override - ) - response = self.post( viewname='checkouts:check_in_document', kwargs={ 'pk': self.test_document.pk } ) self.assertEqual(response.status_code, 302) + self.assertTrue(self.test_document.is_checked_out()) + def test_document_check_in_forcefull_view_with_access(self): + self._create_test_user() + self._check_out_test_document(user=self.test_user) + + self.grant_access( + obj=self.test_document, + permission=permission_document_check_in_override + ) + + response = self.post( + viewname='checkouts:check_in_document', kwargs={ + 'pk': self.test_document.pk + } + ) + self.assertEqual(response.status_code, 302) self.assertFalse(self.test_document.is_checked_out()) diff --git a/mayan/apps/checkouts/urls.py b/mayan/apps/checkouts/urls.py index 994d18edde..0e30989cf4 100644 --- a/mayan/apps/checkouts/urls.py +++ b/mayan/apps/checkouts/urls.py @@ -4,25 +4,34 @@ from django.conf.urls import url from .api_views import APICheckedoutDocumentListView, APICheckedoutDocumentView from .views import ( - CheckoutDocumentView, CheckoutDetailView, CheckoutListView, - DocumentCheckinView + DocumentCheckinView, DocumentCheckoutDetailView, DocumentCheckoutView, + DocumentCheckoutListView ) urlpatterns = [ url( - regex=r'^list/$', view=CheckoutListView.as_view(), name='check_out_list' + regex=r'^documents/$', view=DocumentCheckoutListView.as_view(), + name='check_out_list' ), url( - regex=r'^(?P\d+)/check/out/$', view=CheckoutDocumentView.as_view(), - name='check_out_document' - ), - url( - regex=r'^(?P\d+)/check/in/$', view=DocumentCheckinView.as_view(), + regex=r'^documents/(?P\d+)/check_in/$', view=DocumentCheckinView.as_view(), name='check_in_document' ), url( - regex=r'^(?P\d+)/check/info/$', view=CheckoutDetailView.as_view(), - name='check_out_info' + regex=r'^documents/multiple/check_in/$', + name='check_in_document_multiple', view=DocumentCheckinView.as_view() + ), + url( + regex=r'^documents/(?P\d+)/check_out/$', view=DocumentCheckoutView.as_view(), + name='check_out_document' + ), + url( + regex=r'^documents/multiple/check_out/$', + name='check_out_document_multiple', view=DocumentCheckoutView.as_view() + ), + url( + regex=r'^documents/(?P\d+)/checkout/info/$', + view=DocumentCheckoutDetailView.as_view(), name='check_out_info' ), ] diff --git a/mayan/apps/checkouts/views.py b/mayan/apps/checkouts/views.py index 65063d2cfe..440c73a5f2 100644 --- a/mayan/apps/checkouts/views.py +++ b/mayan/apps/checkouts/views.py @@ -1,20 +1,16 @@ from __future__ import absolute_import, unicode_literals -from django.contrib import messages -from django.http import HttpResponseRedirect -from django.shortcuts import get_object_or_404 from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, ungettext from mayan.apps.acls.models import AccessControlList from mayan.apps.common.generics import ( - ConfirmView, SingleObjectCreateView, SingleObjectDetailView + MultipleObjectConfirmActionView, MultipleObjectFormActionView, + SingleObjectDetailView ) -from mayan.apps.common.utils import encapsulate from mayan.apps.documents.models import Document from mayan.apps.documents.views import DocumentListView -from .exceptions import DocumentAlreadyCheckedOut, DocumentNotCheckedOut from .forms import DocumentCheckoutForm, DocumentCheckoutDefailForm from .icons import icon_check_out_info from .models import DocumentCheckout @@ -24,159 +20,124 @@ from .permissions import ( ) -class DocumentCheckinView(ConfirmView): - def get_extra_context(self): - document = self.get_object() - - context = { - 'object': document, - } - - if document.get_check_out_info().user != self.request.user: - context['title'] = _( - 'You didn\'t originally checked out this document. ' - 'Forcefully check in the document: %s?' - ) % document - else: - context['title'] = _('Check in the document: %s?') % document - - return context - - def get_object(self): - return get_object_or_404(klass=Document, pk=self.kwargs['pk']) - - def get_post_action_redirect(self): - return reverse( - viewname='checkouts:check_out_info', kwargs={ - 'pk': self.get_object().pk - } - ) - - def view_action(self): - document = self.get_object() - - if document.get_check_out_info().user == self.request.user: - AccessControlList.objects.check_access( - obj=document, permissions=(permission_document_check_in,), - user=self.request.user - ) - else: - AccessControlList.objects.check_access( - obj=document, - permissions=(permission_document_check_in_override,), - user=self.request.user - ) - - try: - document.check_in(user=self.request.user) - except DocumentNotCheckedOut: - messages.error( - message=_('Document has not been checked out.'), - request=self.request - ) - else: - messages.success( - message=_( - 'Document "%s" checked in successfully.' - ) % document, request=self.request - ) - - -class CheckoutDocumentView(SingleObjectCreateView): - form_class = DocumentCheckoutForm - - def dispatch(self, request, *args, **kwargs): - self.document = get_object_or_404(klass=Document, pk=self.kwargs['pk']) - - AccessControlList.objects.check_access( - obj=self.document, permissions=(permission_document_check_out,), - user=request.user - ) - - return super( - CheckoutDocumentView, self - ).dispatch(request, *args, **kwargs) - - def form_valid(self, form): - try: - instance = form.save(commit=False) - instance.user = self.request.user - instance.document = self.document - instance.save() - except DocumentAlreadyCheckedOut: - messages.error( - message=_('Document already checked out.'), - request=self.request - ) - else: - messages.success( - message=_( - 'Document "%s" checked out successfully.' - ) % self.document, request=self.request - ) - - return HttpResponseRedirect(redirect_to=self.get_success_url()) +class DocumentCheckinView(MultipleObjectConfirmActionView): + error_message = 'Unable to check in document "%(instance)s". %(exception)s' + model = Document + pk_url_kwarg = 'pk' + success_message_singular = '%(count)d document checked in.' + success_message_plural = '%(count)d documents checked in.' def get_extra_context(self): - return { - 'object': self.document, - 'title': _('Check out document: %s') % self.document + queryset = self.get_object_list() + + result = { + 'title': ungettext( + singular='Check in %(count)d document', + plural='Check in %(count)d documents', + number=queryset.count() + ) % { + 'count': queryset.count(), + } } - def get_post_action_redirect(self): - return reverse( - viewname='checkouts:check_out_info', kwargs={ - 'pk': self.document.pk - } - ) + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Check in document: %s' + ) % queryset.first() + } + ) + return result -class CheckoutListView(DocumentListView): - def get_document_queryset(self): - return AccessControlList.objects.restrict_queryset( - permission=permission_document_check_out_detail_view, - queryset=DocumentCheckout.objects.checked_out_documents(), + def get_post_object_action_url(self): + if self.action_count == 1: + return reverse( + viewname='checkouts:document_checkout_info', + kwargs={'pk': self.action_id_list[0]} + ) + else: + super(DocumentCheckinView, self).get_post_action_redirect() + + def get_source_queryset(self): + # object_permission is None to disable restricting queryset mixin + # and restrict the queryset ourselves from two permissions + + source_queryset = super(DocumentCheckinView, self).get_source_queryset() + + check_in_queryset = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in, queryset=source_queryset, user=self.request.user ) - def get_extra_context(self): - context = super(CheckoutListView, self).get_extra_context() - context.update( - { - 'extra_columns': ( - { - 'name': _('User'), - 'attribute': encapsulate( - lambda document: document.get_check_out_info().user.get_full_name() or document.get_check_out_info().user - ) - }, - { - 'name': _('Checkout time and date'), - 'attribute': encapsulate( - lambda document: document.get_check_out_info().checkout_datetime - ) - }, - { - 'name': _('Checkout expiration'), - 'attribute': encapsulate( - lambda document: document.get_check_out_info().expiration_datetime - ) - }, - ), - 'no_results_icon': icon_check_out_info, - 'no_results_text': _( - 'Checking out a document blocks certain document ' - 'operations for a predetermined amount of ' - 'time.' - ), - 'no_results_title': _('No documents have been checked out'), - 'title': _('Documents checked out'), - } + check_in_override_queryset = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in_override, + queryset=source_queryset, user=self.request.user + ) + + return check_in_queryset | check_in_override_queryset + + def object_action(self, form, instance): + DocumentCheckout.business_logic.check_in_document( + document=instance, user=self.request.user ) - return context -class CheckoutDetailView(SingleObjectDetailView): +class DocumentCheckoutView(MultipleObjectFormActionView): + error_message = 'Unable to checkout document "%(instance)s". %(exception)s' + form_class = DocumentCheckoutForm + model = Document + object_permission = permission_document_check_out + pk_url_kwarg = 'pk' + success_message_singular = '%(count)d document checked out.' + success_message_plural = '%(count)d documents checked out.' + + def get_extra_context(self): + queryset = self.get_object_list() + + result = { + 'title': ungettext( + singular='Checkout %(count)d document', + plural='Checkout %(count)d documents', + number=queryset.count() + ) % { + 'count': queryset.count(), + } + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Check out document: %s' + ) % queryset.first() + } + ) + + return result + + def get_post_object_action_url(self): + if self.action_count == 1: + return reverse( + viewname='checkouts:document_checkout_info', + kwargs={'pk': self.action_id_list[0]} + ) + else: + super(DocumentCheckoutView, self).get_post_action_redirect() + + def object_action(self, form, instance): + DocumentCheckout.objects.check_out_document( + block_new_version=form.cleaned_data['block_new_version'], + document=instance, + expiration_datetime=form.cleaned_data['expiration_datetime'], + user=self.request.user, + ) + + +class DocumentCheckoutDetailView(SingleObjectDetailView): form_class = DocumentCheckoutDefailForm model = Document object_permission = permission_document_check_out_detail_view @@ -188,3 +149,27 @@ class CheckoutDetailView(SingleObjectDetailView): 'Check out details for document: %s' ) % self.object } + + +class DocumentCheckoutListView(DocumentListView): + def get_document_queryset(self): + return AccessControlList.objects.restrict_queryset( + permission=permission_document_check_out_detail_view, + queryset=DocumentCheckout.objects.checked_out_documents(), + user=self.request.user + ) + + def get_extra_context(self): + context = super(DocumentCheckoutListView, self).get_extra_context() + context.update( + { + 'no_results_icon': icon_check_out_info, + 'no_results_text': _( + 'Checking out a document, blocks certain operations ' + 'for a predetermined amount of time.' + ), + 'no_results_title': _('No documents have been checked out'), + 'title': _('Checked out documents'), + } + ) + return context diff --git a/mayan/apps/checkouts/widgets.py b/mayan/apps/checkouts/widgets.py index 72497c8fba..5206cc662e 100644 --- a/mayan/apps/checkouts/widgets.py +++ b/mayan/apps/checkouts/widgets.py @@ -32,8 +32,8 @@ class SplitTimeDeltaWidget(forms.widgets.MultiWidget): return (None, None) def value_from_datadict(self, querydict, files, name): - unit = querydict.get('{}_1'.format(name)) - period = querydict.get('{}_0'.format(name)) + unit = querydict.get('{}_0'.format(name)) + period = querydict.get('{}_1'.format(name)) if not unit or not period: return now() diff --git a/mayan/apps/common/apps.py b/mayan/apps/common/apps.py index f097153661..324f33dc4e 100644 --- a/mayan/apps/common/apps.py +++ b/mayan/apps/common/apps.py @@ -27,9 +27,7 @@ from .links import ( ) from .literals import MESSAGE_SQLITE_WARNING -from .menus import ( - menu_about, menu_main, menu_secondary, menu_topbar, menu_user -) +from .menus import menu_about, menu_secondary, menu_topbar, menu_user from .settings import ( setting_auto_logging, setting_production_error_log_path, setting_production_error_logging diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py deleted file mode 100644 index 30bc2d0e67..0000000000 --- a/mayan/apps/common/management/commands/convertdb.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import unicode_literals - -import errno -import os -import warnings - -from pathlib2 import Path - -from django.conf import settings -from django.core import management -from django.core.management.base import CommandError -from django.utils.encoding import force_text -from django.utils.translation import ugettext_lazy as _ - -from mayan.apps.documents.models import DocumentType -from mayan.apps.storage.utils import fs_cleanup - -from ...literals import MESSAGE_DEPRECATION_WARNING -from ...warnings import DeprecationWarning - -CONVERTDB_FOLDER = 'convertdb' -CONVERTDB_OUTPUT_FILENAME = 'migrate.json' - - -class Command(management.BaseCommand): - help = 'Convert from a database backend to another one.' - - def __init__(self, *args, **kwargs): - warnings.warn( - category=DeprecationWarning, - message=force_text(MESSAGE_DEPRECATION_WARNING) - ) - - super(Command, self).__init__(*args, **kwargs) - - def add_arguments(self, parser): - parser.add_argument( - 'args', metavar='app_label[.ModelName]', nargs='*', - help=_( - 'Restricts dumped data to the specified app_label or ' - 'app_label.ModelName.' - ) - ) - parser.add_argument( - '--from', action='store', default='default', dest='from', - help=_( - 'The database from which data will be exported. If omitted ' - 'the database named "default" will be used.' - ), - ) - parser.add_argument( - '--to', action='store', default='default', dest='to', - help=_( - 'The database to which data will be imported. If omitted ' - 'the database named "default" will be used.' - ), - ) - parser.add_argument( - '--force', action='store_true', dest='force', - help=_( - 'Force the conversion of the database even if the receiving ' - 'database is not empty.' - ), - ) - - def handle(self, *app_labels, **options): - # Create the media/convertdb folder - convertdb_folder_path = force_text( - Path( - settings.MEDIA_ROOT, CONVERTDB_FOLDER - ) - ) - - try: - os.makedirs(convertdb_folder_path) - except OSError as exception: - if exception.errno == errno.EEXIST: - pass - - convertdb_file_path = force_text( - Path( - convertdb_folder_path, CONVERTDB_OUTPUT_FILENAME - ) - ) - - management.call_command(command_name='purgeperiodictasks') - - management.call_command( - 'dumpdata', *app_labels, all=True, - database=options['from'], natural_primary=True, - natural_foreign=True, output=convertdb_file_path, - interactive=False, format='json' - ) - - if DocumentType.objects.using(options['to']).count() and not options['force']: - fs_cleanup(convertdb_file_path) - raise CommandError( - 'There is existing data in the database that will be ' - 'used for the import. If you proceed with the conversion ' - 'you might lose data. Please check your settings.' - ) - - management.call_command( - 'loaddata', convertdb_file_path, database=options['to'], interactive=False, - verbosity=3 - ) - fs_cleanup(convertdb_file_path) diff --git a/mayan/apps/common/migrations/0012_auto_20190711_0548.py b/mayan/apps/common/migrations/0012_auto_20190711_0548.py new file mode 100644 index 0000000000..9915d05e2c --- /dev/null +++ b/mayan/apps/common/migrations/0012_auto_20190711_0548.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-07-11 05:48 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.common.models +import mayan.apps.storage.classes + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0011_auto_20180429_0758'), + ] + + operations = [ + migrations.AlterField( + model_name='shareduploadedfile', + name='file', + field=models.FileField(storage=mayan.apps.storage.classes.FakeStorageSubclass(), upload_to=mayan.apps.common.models.upload_to, verbose_name='File'), + ), + migrations.AlterField( + model_name='userlocaleprofile', + name='language', + field=models.CharField(choices=[('ar', 'Arabic'), ('bg', 'Bulgarian'), ('bs', 'Bosnian'), ('cs', 'Czech'), ('da', 'Danish'), ('de', 'German'), ('el', 'Greek'), ('en', 'English'), ('es', 'Spanish'), ('fa', 'Persian'), ('fr', 'French'), ('hu', 'Hungarian'), ('id', 'Indonesian'), ('it', 'Italian'), ('lv', 'Latvian'), ('nl', 'Dutch'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('pt-br', 'Portuguese (Brazil)'), ('ro', 'Romanian'), ('ru', 'Russian'), ('sl', 'Slovenian'), ('tr', 'Turkish'), ('vi', 'Vietnamese'), ('zh', 'Chinese')], max_length=8, verbose_name='Language'), + ), + migrations.AlterField( + model_name='userlocaleprofile', + name='timezone', + field=models.CharField(choices=[(b'Africa/Abidjan', b'Africa/Abidjan'), (b'Africa/Accra', b'Africa/Accra'), (b'Africa/Addis_Ababa', b'Africa/Addis_Ababa'), (b'Africa/Algiers', b'Africa/Algiers'), (b'Africa/Asmara', b'Africa/Asmara'), (b'Africa/Bamako', b'Africa/Bamako'), (b'Africa/Bangui', b'Africa/Bangui'), (b'Africa/Banjul', b'Africa/Banjul'), (b'Africa/Bissau', b'Africa/Bissau'), (b'Africa/Blantyre', b'Africa/Blantyre'), (b'Africa/Brazzaville', b'Africa/Brazzaville'), (b'Africa/Bujumbura', b'Africa/Bujumbura'), (b'Africa/Cairo', b'Africa/Cairo'), (b'Africa/Casablanca', b'Africa/Casablanca'), (b'Africa/Ceuta', b'Africa/Ceuta'), (b'Africa/Conakry', b'Africa/Conakry'), (b'Africa/Dakar', b'Africa/Dakar'), (b'Africa/Dar_es_Salaam', b'Africa/Dar_es_Salaam'), (b'Africa/Djibouti', b'Africa/Djibouti'), (b'Africa/Douala', b'Africa/Douala'), (b'Africa/El_Aaiun', b'Africa/El_Aaiun'), (b'Africa/Freetown', b'Africa/Freetown'), (b'Africa/Gaborone', b'Africa/Gaborone'), (b'Africa/Harare', b'Africa/Harare'), (b'Africa/Johannesburg', b'Africa/Johannesburg'), (b'Africa/Juba', b'Africa/Juba'), (b'Africa/Kampala', b'Africa/Kampala'), (b'Africa/Khartoum', b'Africa/Khartoum'), (b'Africa/Kigali', b'Africa/Kigali'), (b'Africa/Kinshasa', b'Africa/Kinshasa'), (b'Africa/Lagos', b'Africa/Lagos'), (b'Africa/Libreville', b'Africa/Libreville'), (b'Africa/Lome', b'Africa/Lome'), (b'Africa/Luanda', b'Africa/Luanda'), (b'Africa/Lubumbashi', b'Africa/Lubumbashi'), (b'Africa/Lusaka', b'Africa/Lusaka'), (b'Africa/Malabo', b'Africa/Malabo'), (b'Africa/Maputo', b'Africa/Maputo'), (b'Africa/Maseru', b'Africa/Maseru'), (b'Africa/Mbabane', b'Africa/Mbabane'), (b'Africa/Mogadishu', b'Africa/Mogadishu'), (b'Africa/Monrovia', b'Africa/Monrovia'), (b'Africa/Nairobi', b'Africa/Nairobi'), (b'Africa/Ndjamena', b'Africa/Ndjamena'), (b'Africa/Niamey', b'Africa/Niamey'), (b'Africa/Nouakchott', b'Africa/Nouakchott'), (b'Africa/Ouagadougou', b'Africa/Ouagadougou'), (b'Africa/Porto-Novo', b'Africa/Porto-Novo'), (b'Africa/Sao_Tome', b'Africa/Sao_Tome'), (b'Africa/Tripoli', b'Africa/Tripoli'), (b'Africa/Tunis', b'Africa/Tunis'), (b'Africa/Windhoek', b'Africa/Windhoek'), (b'America/Adak', b'America/Adak'), (b'America/Anchorage', b'America/Anchorage'), (b'America/Anguilla', b'America/Anguilla'), (b'America/Antigua', b'America/Antigua'), (b'America/Araguaina', b'America/Araguaina'), (b'America/Argentina/Buenos_Aires', b'America/Argentina/Buenos_Aires'), (b'America/Argentina/Catamarca', b'America/Argentina/Catamarca'), (b'America/Argentina/Cordoba', b'America/Argentina/Cordoba'), (b'America/Argentina/Jujuy', b'America/Argentina/Jujuy'), (b'America/Argentina/La_Rioja', b'America/Argentina/La_Rioja'), (b'America/Argentina/Mendoza', b'America/Argentina/Mendoza'), (b'America/Argentina/Rio_Gallegos', b'America/Argentina/Rio_Gallegos'), (b'America/Argentina/Salta', b'America/Argentina/Salta'), (b'America/Argentina/San_Juan', b'America/Argentina/San_Juan'), (b'America/Argentina/San_Luis', b'America/Argentina/San_Luis'), (b'America/Argentina/Tucuman', b'America/Argentina/Tucuman'), (b'America/Argentina/Ushuaia', b'America/Argentina/Ushuaia'), (b'America/Aruba', b'America/Aruba'), (b'America/Asuncion', b'America/Asuncion'), (b'America/Atikokan', b'America/Atikokan'), (b'America/Bahia', b'America/Bahia'), (b'America/Bahia_Banderas', b'America/Bahia_Banderas'), (b'America/Barbados', b'America/Barbados'), (b'America/Belem', b'America/Belem'), (b'America/Belize', b'America/Belize'), (b'America/Blanc-Sablon', b'America/Blanc-Sablon'), (b'America/Boa_Vista', b'America/Boa_Vista'), (b'America/Bogota', b'America/Bogota'), (b'America/Boise', b'America/Boise'), (b'America/Cambridge_Bay', b'America/Cambridge_Bay'), (b'America/Campo_Grande', b'America/Campo_Grande'), (b'America/Cancun', b'America/Cancun'), (b'America/Caracas', b'America/Caracas'), (b'America/Cayenne', b'America/Cayenne'), (b'America/Cayman', b'America/Cayman'), (b'America/Chicago', b'America/Chicago'), (b'America/Chihuahua', b'America/Chihuahua'), (b'America/Costa_Rica', b'America/Costa_Rica'), (b'America/Creston', b'America/Creston'), (b'America/Cuiaba', b'America/Cuiaba'), (b'America/Curacao', b'America/Curacao'), (b'America/Danmarkshavn', b'America/Danmarkshavn'), (b'America/Dawson', b'America/Dawson'), (b'America/Dawson_Creek', b'America/Dawson_Creek'), (b'America/Denver', b'America/Denver'), (b'America/Detroit', b'America/Detroit'), (b'America/Dominica', b'America/Dominica'), (b'America/Edmonton', b'America/Edmonton'), (b'America/Eirunepe', b'America/Eirunepe'), (b'America/El_Salvador', b'America/El_Salvador'), (b'America/Fort_Nelson', b'America/Fort_Nelson'), (b'America/Fortaleza', b'America/Fortaleza'), (b'America/Glace_Bay', b'America/Glace_Bay'), (b'America/Godthab', b'America/Godthab'), (b'America/Goose_Bay', b'America/Goose_Bay'), (b'America/Grand_Turk', b'America/Grand_Turk'), (b'America/Grenada', b'America/Grenada'), (b'America/Guadeloupe', b'America/Guadeloupe'), (b'America/Guatemala', b'America/Guatemala'), (b'America/Guayaquil', b'America/Guayaquil'), (b'America/Guyana', b'America/Guyana'), (b'America/Halifax', b'America/Halifax'), (b'America/Havana', b'America/Havana'), (b'America/Hermosillo', b'America/Hermosillo'), (b'America/Indiana/Indianapolis', b'America/Indiana/Indianapolis'), (b'America/Indiana/Knox', b'America/Indiana/Knox'), (b'America/Indiana/Marengo', b'America/Indiana/Marengo'), (b'America/Indiana/Petersburg', b'America/Indiana/Petersburg'), (b'America/Indiana/Tell_City', b'America/Indiana/Tell_City'), (b'America/Indiana/Vevay', b'America/Indiana/Vevay'), (b'America/Indiana/Vincennes', b'America/Indiana/Vincennes'), (b'America/Indiana/Winamac', b'America/Indiana/Winamac'), (b'America/Inuvik', b'America/Inuvik'), (b'America/Iqaluit', b'America/Iqaluit'), (b'America/Jamaica', b'America/Jamaica'), (b'America/Juneau', b'America/Juneau'), (b'America/Kentucky/Louisville', b'America/Kentucky/Louisville'), (b'America/Kentucky/Monticello', b'America/Kentucky/Monticello'), (b'America/Kralendijk', b'America/Kralendijk'), (b'America/La_Paz', b'America/La_Paz'), (b'America/Lima', b'America/Lima'), (b'America/Los_Angeles', b'America/Los_Angeles'), (b'America/Lower_Princes', b'America/Lower_Princes'), (b'America/Maceio', b'America/Maceio'), (b'America/Managua', b'America/Managua'), (b'America/Manaus', b'America/Manaus'), (b'America/Marigot', b'America/Marigot'), (b'America/Martinique', b'America/Martinique'), (b'America/Matamoros', b'America/Matamoros'), (b'America/Mazatlan', b'America/Mazatlan'), (b'America/Menominee', b'America/Menominee'), (b'America/Merida', b'America/Merida'), (b'America/Metlakatla', b'America/Metlakatla'), (b'America/Mexico_City', b'America/Mexico_City'), (b'America/Miquelon', b'America/Miquelon'), (b'America/Moncton', b'America/Moncton'), (b'America/Monterrey', b'America/Monterrey'), (b'America/Montevideo', b'America/Montevideo'), (b'America/Montserrat', b'America/Montserrat'), (b'America/Nassau', b'America/Nassau'), (b'America/New_York', b'America/New_York'), (b'America/Nipigon', b'America/Nipigon'), (b'America/Nome', b'America/Nome'), (b'America/Noronha', b'America/Noronha'), (b'America/North_Dakota/Beulah', b'America/North_Dakota/Beulah'), (b'America/North_Dakota/Center', b'America/North_Dakota/Center'), (b'America/North_Dakota/New_Salem', b'America/North_Dakota/New_Salem'), (b'America/Ojinaga', b'America/Ojinaga'), (b'America/Panama', b'America/Panama'), (b'America/Pangnirtung', b'America/Pangnirtung'), (b'America/Paramaribo', b'America/Paramaribo'), (b'America/Phoenix', b'America/Phoenix'), (b'America/Port-au-Prince', b'America/Port-au-Prince'), (b'America/Port_of_Spain', b'America/Port_of_Spain'), (b'America/Porto_Velho', b'America/Porto_Velho'), (b'America/Puerto_Rico', b'America/Puerto_Rico'), (b'America/Punta_Arenas', b'America/Punta_Arenas'), (b'America/Rainy_River', b'America/Rainy_River'), (b'America/Rankin_Inlet', b'America/Rankin_Inlet'), (b'America/Recife', b'America/Recife'), (b'America/Regina', b'America/Regina'), (b'America/Resolute', b'America/Resolute'), (b'America/Rio_Branco', b'America/Rio_Branco'), (b'America/Santarem', b'America/Santarem'), (b'America/Santiago', b'America/Santiago'), (b'America/Santo_Domingo', b'America/Santo_Domingo'), (b'America/Sao_Paulo', b'America/Sao_Paulo'), (b'America/Scoresbysund', b'America/Scoresbysund'), (b'America/Sitka', b'America/Sitka'), (b'America/St_Barthelemy', b'America/St_Barthelemy'), (b'America/St_Johns', b'America/St_Johns'), (b'America/St_Kitts', b'America/St_Kitts'), (b'America/St_Lucia', b'America/St_Lucia'), (b'America/St_Thomas', b'America/St_Thomas'), (b'America/St_Vincent', b'America/St_Vincent'), (b'America/Swift_Current', b'America/Swift_Current'), (b'America/Tegucigalpa', b'America/Tegucigalpa'), (b'America/Thule', b'America/Thule'), (b'America/Thunder_Bay', b'America/Thunder_Bay'), (b'America/Tijuana', b'America/Tijuana'), (b'America/Toronto', b'America/Toronto'), (b'America/Tortola', b'America/Tortola'), (b'America/Vancouver', b'America/Vancouver'), (b'America/Whitehorse', b'America/Whitehorse'), (b'America/Winnipeg', b'America/Winnipeg'), (b'America/Yakutat', b'America/Yakutat'), (b'America/Yellowknife', b'America/Yellowknife'), (b'Antarctica/Casey', b'Antarctica/Casey'), (b'Antarctica/Davis', b'Antarctica/Davis'), (b'Antarctica/DumontDUrville', b'Antarctica/DumontDUrville'), (b'Antarctica/Macquarie', b'Antarctica/Macquarie'), (b'Antarctica/Mawson', b'Antarctica/Mawson'), (b'Antarctica/McMurdo', b'Antarctica/McMurdo'), (b'Antarctica/Palmer', b'Antarctica/Palmer'), (b'Antarctica/Rothera', b'Antarctica/Rothera'), (b'Antarctica/Syowa', b'Antarctica/Syowa'), (b'Antarctica/Troll', b'Antarctica/Troll'), (b'Antarctica/Vostok', b'Antarctica/Vostok'), (b'Arctic/Longyearbyen', b'Arctic/Longyearbyen'), (b'Asia/Aden', b'Asia/Aden'), (b'Asia/Almaty', b'Asia/Almaty'), (b'Asia/Amman', b'Asia/Amman'), (b'Asia/Anadyr', b'Asia/Anadyr'), (b'Asia/Aqtau', b'Asia/Aqtau'), (b'Asia/Aqtobe', b'Asia/Aqtobe'), (b'Asia/Ashgabat', b'Asia/Ashgabat'), (b'Asia/Atyrau', b'Asia/Atyrau'), (b'Asia/Baghdad', b'Asia/Baghdad'), (b'Asia/Bahrain', b'Asia/Bahrain'), (b'Asia/Baku', b'Asia/Baku'), (b'Asia/Bangkok', b'Asia/Bangkok'), (b'Asia/Barnaul', b'Asia/Barnaul'), (b'Asia/Beirut', b'Asia/Beirut'), (b'Asia/Bishkek', b'Asia/Bishkek'), (b'Asia/Brunei', b'Asia/Brunei'), (b'Asia/Chita', b'Asia/Chita'), (b'Asia/Choibalsan', b'Asia/Choibalsan'), (b'Asia/Colombo', b'Asia/Colombo'), (b'Asia/Damascus', b'Asia/Damascus'), (b'Asia/Dhaka', b'Asia/Dhaka'), (b'Asia/Dili', b'Asia/Dili'), (b'Asia/Dubai', b'Asia/Dubai'), (b'Asia/Dushanbe', b'Asia/Dushanbe'), (b'Asia/Famagusta', b'Asia/Famagusta'), (b'Asia/Gaza', b'Asia/Gaza'), (b'Asia/Hebron', b'Asia/Hebron'), (b'Asia/Ho_Chi_Minh', b'Asia/Ho_Chi_Minh'), (b'Asia/Hong_Kong', b'Asia/Hong_Kong'), (b'Asia/Hovd', b'Asia/Hovd'), (b'Asia/Irkutsk', b'Asia/Irkutsk'), (b'Asia/Jakarta', b'Asia/Jakarta'), (b'Asia/Jayapura', b'Asia/Jayapura'), (b'Asia/Jerusalem', b'Asia/Jerusalem'), (b'Asia/Kabul', b'Asia/Kabul'), (b'Asia/Kamchatka', b'Asia/Kamchatka'), (b'Asia/Karachi', b'Asia/Karachi'), (b'Asia/Kathmandu', b'Asia/Kathmandu'), (b'Asia/Khandyga', b'Asia/Khandyga'), (b'Asia/Kolkata', b'Asia/Kolkata'), (b'Asia/Krasnoyarsk', b'Asia/Krasnoyarsk'), (b'Asia/Kuala_Lumpur', b'Asia/Kuala_Lumpur'), (b'Asia/Kuching', b'Asia/Kuching'), (b'Asia/Kuwait', b'Asia/Kuwait'), (b'Asia/Macau', b'Asia/Macau'), (b'Asia/Magadan', b'Asia/Magadan'), (b'Asia/Makassar', b'Asia/Makassar'), (b'Asia/Manila', b'Asia/Manila'), (b'Asia/Muscat', b'Asia/Muscat'), (b'Asia/Nicosia', b'Asia/Nicosia'), (b'Asia/Novokuznetsk', b'Asia/Novokuznetsk'), (b'Asia/Novosibirsk', b'Asia/Novosibirsk'), (b'Asia/Omsk', b'Asia/Omsk'), (b'Asia/Oral', b'Asia/Oral'), (b'Asia/Phnom_Penh', b'Asia/Phnom_Penh'), (b'Asia/Pontianak', b'Asia/Pontianak'), (b'Asia/Pyongyang', b'Asia/Pyongyang'), (b'Asia/Qatar', b'Asia/Qatar'), (b'Asia/Qostanay', b'Asia/Qostanay'), (b'Asia/Qyzylorda', b'Asia/Qyzylorda'), (b'Asia/Riyadh', b'Asia/Riyadh'), (b'Asia/Sakhalin', b'Asia/Sakhalin'), (b'Asia/Samarkand', b'Asia/Samarkand'), (b'Asia/Seoul', b'Asia/Seoul'), (b'Asia/Shanghai', b'Asia/Shanghai'), (b'Asia/Singapore', b'Asia/Singapore'), (b'Asia/Srednekolymsk', b'Asia/Srednekolymsk'), (b'Asia/Taipei', b'Asia/Taipei'), (b'Asia/Tashkent', b'Asia/Tashkent'), (b'Asia/Tbilisi', b'Asia/Tbilisi'), (b'Asia/Tehran', b'Asia/Tehran'), (b'Asia/Thimphu', b'Asia/Thimphu'), (b'Asia/Tokyo', b'Asia/Tokyo'), (b'Asia/Tomsk', b'Asia/Tomsk'), (b'Asia/Ulaanbaatar', b'Asia/Ulaanbaatar'), (b'Asia/Urumqi', b'Asia/Urumqi'), (b'Asia/Ust-Nera', b'Asia/Ust-Nera'), (b'Asia/Vientiane', b'Asia/Vientiane'), (b'Asia/Vladivostok', b'Asia/Vladivostok'), (b'Asia/Yakutsk', b'Asia/Yakutsk'), (b'Asia/Yangon', b'Asia/Yangon'), (b'Asia/Yekaterinburg', b'Asia/Yekaterinburg'), (b'Asia/Yerevan', b'Asia/Yerevan'), (b'Atlantic/Azores', b'Atlantic/Azores'), (b'Atlantic/Bermuda', b'Atlantic/Bermuda'), (b'Atlantic/Canary', b'Atlantic/Canary'), (b'Atlantic/Cape_Verde', b'Atlantic/Cape_Verde'), (b'Atlantic/Faroe', b'Atlantic/Faroe'), (b'Atlantic/Madeira', b'Atlantic/Madeira'), (b'Atlantic/Reykjavik', b'Atlantic/Reykjavik'), (b'Atlantic/South_Georgia', b'Atlantic/South_Georgia'), (b'Atlantic/St_Helena', b'Atlantic/St_Helena'), (b'Atlantic/Stanley', b'Atlantic/Stanley'), (b'Australia/Adelaide', b'Australia/Adelaide'), (b'Australia/Brisbane', b'Australia/Brisbane'), (b'Australia/Broken_Hill', b'Australia/Broken_Hill'), (b'Australia/Currie', b'Australia/Currie'), (b'Australia/Darwin', b'Australia/Darwin'), (b'Australia/Eucla', b'Australia/Eucla'), (b'Australia/Hobart', b'Australia/Hobart'), (b'Australia/Lindeman', b'Australia/Lindeman'), (b'Australia/Lord_Howe', b'Australia/Lord_Howe'), (b'Australia/Melbourne', b'Australia/Melbourne'), (b'Australia/Perth', b'Australia/Perth'), (b'Australia/Sydney', b'Australia/Sydney'), (b'Canada/Atlantic', b'Canada/Atlantic'), (b'Canada/Central', b'Canada/Central'), (b'Canada/Eastern', b'Canada/Eastern'), (b'Canada/Mountain', b'Canada/Mountain'), (b'Canada/Newfoundland', b'Canada/Newfoundland'), (b'Canada/Pacific', b'Canada/Pacific'), (b'Europe/Amsterdam', b'Europe/Amsterdam'), (b'Europe/Andorra', b'Europe/Andorra'), (b'Europe/Astrakhan', b'Europe/Astrakhan'), (b'Europe/Athens', b'Europe/Athens'), (b'Europe/Belgrade', b'Europe/Belgrade'), (b'Europe/Berlin', b'Europe/Berlin'), (b'Europe/Bratislava', b'Europe/Bratislava'), (b'Europe/Brussels', b'Europe/Brussels'), (b'Europe/Bucharest', b'Europe/Bucharest'), (b'Europe/Budapest', b'Europe/Budapest'), (b'Europe/Busingen', b'Europe/Busingen'), (b'Europe/Chisinau', b'Europe/Chisinau'), (b'Europe/Copenhagen', b'Europe/Copenhagen'), (b'Europe/Dublin', b'Europe/Dublin'), (b'Europe/Gibraltar', b'Europe/Gibraltar'), (b'Europe/Guernsey', b'Europe/Guernsey'), (b'Europe/Helsinki', b'Europe/Helsinki'), (b'Europe/Isle_of_Man', b'Europe/Isle_of_Man'), (b'Europe/Istanbul', b'Europe/Istanbul'), (b'Europe/Jersey', b'Europe/Jersey'), (b'Europe/Kaliningrad', b'Europe/Kaliningrad'), (b'Europe/Kiev', b'Europe/Kiev'), (b'Europe/Kirov', b'Europe/Kirov'), (b'Europe/Lisbon', b'Europe/Lisbon'), (b'Europe/Ljubljana', b'Europe/Ljubljana'), (b'Europe/London', b'Europe/London'), (b'Europe/Luxembourg', b'Europe/Luxembourg'), (b'Europe/Madrid', b'Europe/Madrid'), (b'Europe/Malta', b'Europe/Malta'), (b'Europe/Mariehamn', b'Europe/Mariehamn'), (b'Europe/Minsk', b'Europe/Minsk'), (b'Europe/Monaco', b'Europe/Monaco'), (b'Europe/Moscow', b'Europe/Moscow'), (b'Europe/Oslo', b'Europe/Oslo'), (b'Europe/Paris', b'Europe/Paris'), (b'Europe/Podgorica', b'Europe/Podgorica'), (b'Europe/Prague', b'Europe/Prague'), (b'Europe/Riga', b'Europe/Riga'), (b'Europe/Rome', b'Europe/Rome'), (b'Europe/Samara', b'Europe/Samara'), (b'Europe/San_Marino', b'Europe/San_Marino'), (b'Europe/Sarajevo', b'Europe/Sarajevo'), (b'Europe/Saratov', b'Europe/Saratov'), (b'Europe/Simferopol', b'Europe/Simferopol'), (b'Europe/Skopje', b'Europe/Skopje'), (b'Europe/Sofia', b'Europe/Sofia'), (b'Europe/Stockholm', b'Europe/Stockholm'), (b'Europe/Tallinn', b'Europe/Tallinn'), (b'Europe/Tirane', b'Europe/Tirane'), (b'Europe/Ulyanovsk', b'Europe/Ulyanovsk'), (b'Europe/Uzhgorod', b'Europe/Uzhgorod'), (b'Europe/Vaduz', b'Europe/Vaduz'), (b'Europe/Vatican', b'Europe/Vatican'), (b'Europe/Vienna', b'Europe/Vienna'), (b'Europe/Vilnius', b'Europe/Vilnius'), (b'Europe/Volgograd', b'Europe/Volgograd'), (b'Europe/Warsaw', b'Europe/Warsaw'), (b'Europe/Zagreb', b'Europe/Zagreb'), (b'Europe/Zaporozhye', b'Europe/Zaporozhye'), (b'Europe/Zurich', b'Europe/Zurich'), (b'GMT', b'GMT'), (b'Indian/Antananarivo', b'Indian/Antananarivo'), (b'Indian/Chagos', b'Indian/Chagos'), (b'Indian/Christmas', b'Indian/Christmas'), (b'Indian/Cocos', b'Indian/Cocos'), (b'Indian/Comoro', b'Indian/Comoro'), (b'Indian/Kerguelen', b'Indian/Kerguelen'), (b'Indian/Mahe', b'Indian/Mahe'), (b'Indian/Maldives', b'Indian/Maldives'), (b'Indian/Mauritius', b'Indian/Mauritius'), (b'Indian/Mayotte', b'Indian/Mayotte'), (b'Indian/Reunion', b'Indian/Reunion'), (b'Pacific/Apia', b'Pacific/Apia'), (b'Pacific/Auckland', b'Pacific/Auckland'), (b'Pacific/Bougainville', b'Pacific/Bougainville'), (b'Pacific/Chatham', b'Pacific/Chatham'), (b'Pacific/Chuuk', b'Pacific/Chuuk'), (b'Pacific/Easter', b'Pacific/Easter'), (b'Pacific/Efate', b'Pacific/Efate'), (b'Pacific/Enderbury', b'Pacific/Enderbury'), (b'Pacific/Fakaofo', b'Pacific/Fakaofo'), (b'Pacific/Fiji', b'Pacific/Fiji'), (b'Pacific/Funafuti', b'Pacific/Funafuti'), (b'Pacific/Galapagos', b'Pacific/Galapagos'), (b'Pacific/Gambier', b'Pacific/Gambier'), (b'Pacific/Guadalcanal', b'Pacific/Guadalcanal'), (b'Pacific/Guam', b'Pacific/Guam'), (b'Pacific/Honolulu', b'Pacific/Honolulu'), (b'Pacific/Kiritimati', b'Pacific/Kiritimati'), (b'Pacific/Kosrae', b'Pacific/Kosrae'), (b'Pacific/Kwajalein', b'Pacific/Kwajalein'), (b'Pacific/Majuro', b'Pacific/Majuro'), (b'Pacific/Marquesas', b'Pacific/Marquesas'), (b'Pacific/Midway', b'Pacific/Midway'), (b'Pacific/Nauru', b'Pacific/Nauru'), (b'Pacific/Niue', b'Pacific/Niue'), (b'Pacific/Norfolk', b'Pacific/Norfolk'), (b'Pacific/Noumea', b'Pacific/Noumea'), (b'Pacific/Pago_Pago', b'Pacific/Pago_Pago'), (b'Pacific/Palau', b'Pacific/Palau'), (b'Pacific/Pitcairn', b'Pacific/Pitcairn'), (b'Pacific/Pohnpei', b'Pacific/Pohnpei'), (b'Pacific/Port_Moresby', b'Pacific/Port_Moresby'), (b'Pacific/Rarotonga', b'Pacific/Rarotonga'), (b'Pacific/Saipan', b'Pacific/Saipan'), (b'Pacific/Tahiti', b'Pacific/Tahiti'), (b'Pacific/Tarawa', b'Pacific/Tarawa'), (b'Pacific/Tongatapu', b'Pacific/Tongatapu'), (b'Pacific/Wake', b'Pacific/Wake'), (b'Pacific/Wallis', b'Pacific/Wallis'), (b'US/Alaska', b'US/Alaska'), (b'US/Arizona', b'US/Arizona'), (b'US/Central', b'US/Central'), (b'US/Eastern', b'US/Eastern'), (b'US/Hawaii', b'US/Hawaii'), (b'US/Mountain', b'US/Mountain'), (b'US/Pacific', b'US/Pacific'), (b'UTC', b'UTC')], max_length=48, verbose_name='Timezone'), + ), + ] diff --git a/mayan/apps/common/serialization.py b/mayan/apps/common/serialization.py new file mode 100644 index 0000000000..27f9b52c06 --- /dev/null +++ b/mayan/apps/common/serialization.py @@ -0,0 +1,22 @@ +from __future__ import unicode_literals + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +def yaml_dump(*args, **kwargs): + defaults = {'Dumper': SafeDumper} + defaults.update(kwargs) + + return yaml.dump(*args, **defaults) + + +def yaml_load(*args, **kwargs): + defaults = {'Loader': SafeLoader} + defaults.update(kwargs) + + return yaml.load(*args, **defaults) diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 841549b38f..c845cd0a6e 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -6,11 +6,10 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ import mayan -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_COMMON_HOME_VIEW - namespace = Namespace(label=_('Common'), name='common') setting_auto_logging = namespace.add_setting( @@ -95,322 +94,5 @@ setting_shared_storage = namespace.add_setting( ) setting_shared_storage_arguments = namespace.add_setting( global_name='COMMON_SHARED_STORAGE_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'shared_files') - ), quoted=True -) - -namespace = Namespace(label=_('Django'), name='django') - -setting_django_allowed_hosts = namespace.add_setting( - global_name='ALLOWED_HOSTS', default=settings.ALLOWED_HOSTS, - help_text=_( - 'A list of strings representing the host/domain names that this site ' - 'can serve. This is a security measure to prevent HTTP Host header ' - 'attacks, which are possible even under many seemingly-safe web ' - 'server configurations. Values in this list can be ' - 'fully qualified names (e.g. \'www.example.com\'), in which case ' - 'they will be matched against the request\'s Host header exactly ' - '(case-insensitive, not including port). A value beginning with a ' - 'period can be used as a subdomain wildcard: \'.example.com\' will ' - 'match example.com, www.example.com, and any other subdomain of ' - 'example.com. A value of \'*\' will match anything; in this case you ' - 'are responsible to provide your own validation of the Host header ' - '(perhaps in a middleware; if so this middleware must be listed ' - 'first in MIDDLEWARE).' - ), -) -setting_django_append_slash = namespace.add_setting( - global_name='APPEND_SLASH', default=settings.APPEND_SLASH, - help_text=_( - 'When set to True, if the request URL does not match any of the ' - 'patterns in the URLconf and it doesn\'t end in a slash, an HTTP ' - 'redirect is issued to the same URL with a slash appended. Note ' - 'that the redirect may cause any data submitted in a POST request ' - 'to be lost. The APPEND_SLASH setting is only used if ' - 'CommonMiddleware is installed (see Middleware). See also ' - 'PREPEND_WWW.' - ) -) -setting_django_auth_password_validators = namespace.add_setting( - global_name='AUTH_PASSWORD_VALIDATORS', - default=settings.AUTH_PASSWORD_VALIDATORS, - help_text=_( - 'The list of validators that are used to check the strength of ' - 'user\'s passwords.' - ) -) -setting_django_databases = namespace.add_setting( - global_name='DATABASES', default=settings.DATABASES, - help_text=_( - 'A dictionary containing the settings for all databases to be used ' - 'with Django. It is a nested dictionary whose contents map a ' - 'database alias to a dictionary containing the options for an ' - 'individual database. The DATABASES setting must configure a ' - 'default database; any number of additional databases may also ' - 'be specified.' - ), -) -setting_django_data_upload_max_memory_size = namespace.add_setting( - global_name='DATA_UPLOAD_MAX_MEMORY_SIZE', - default=settings.DATA_UPLOAD_MAX_MEMORY_SIZE, - help_text=_( - 'Default: 2621440 (i.e. 2.5 MB). The maximum size in bytes that a ' - 'request body may be before a SuspiciousOperation ' - '(RequestDataTooBig) is raised. The check is done when accessing ' - 'request.body or request.POST and is calculated against the total ' - 'request size excluding any file upload data. You can set this to ' - 'None to disable the check. Applications that are expected to ' - 'receive unusually large form posts should tune this setting. The ' - 'amount of request data is correlated to the amount of memory ' - 'needed to process the request and populate the GET and POST ' - 'dictionaries. Large requests could be used as a ' - 'denial-of-service attack vector if left unchecked. Since web ' - 'servers don\'t typically perform deep request inspection, it\'s ' - 'not possible to perform a similar check at that level. See also ' - 'FILE_UPLOAD_MAX_MEMORY_SIZE.' - ), -) -setting_django_default_from_email = namespace.add_setting( - global_name='DEFAULT_FROM_EMAIL', - default=settings.DEFAULT_FROM_EMAIL, - help_text=_( - 'Default: \'webmaster@localhost\' ' - 'Default email address to use for various automated correspondence ' - 'from the site manager(s). This doesn\'t include error messages sent ' - 'to ADMINS and MANAGERS; for that, see SERVER_EMAIL.' - ), -) -setting_django_disallowed_user_agents = namespace.add_setting( - global_name='DISALLOWED_USER_AGENTS', - default=settings.DISALLOWED_USER_AGENTS, - help_text=_( - 'Default: [] (Empty list). List of compiled regular expression ' - 'objects representing User-Agent strings that are not allowed to ' - 'visit any page, systemwide. Use this for bad robots/crawlers. ' - 'This is only used if CommonMiddleware is installed ' - '(see Middleware).' - ), -) -setting_django_email_backend = namespace.add_setting( - global_name='EMAIL_BACKEND', - default=settings.EMAIL_BACKEND, - help_text=_( - 'Default: \'django.core.mail.backends.smtp.EmailBackend\'. The ' - 'backend to use for sending emails.' - ), -) -setting_django_email_host = namespace.add_setting( - global_name='EMAIL_HOST', - default=settings.EMAIL_HOST, - help_text=_( - 'Default: \'localhost\'. The host to use for sending email.' - ), -) -setting_django_email_host_password = namespace.add_setting( - global_name='EMAIL_HOST_PASSWORD', - default=settings.EMAIL_HOST_PASSWORD, - help_text=_( - 'Default: \'\' (Empty string). Password to use for the SMTP ' - 'server defined in EMAIL_HOST. This setting is used in ' - 'conjunction with EMAIL_HOST_USER when authenticating to the ' - 'SMTP server. If either of these settings is empty, ' - 'Django won\'t attempt authentication.' - ), -) -setting_django_email_host_user = namespace.add_setting( - global_name='EMAIL_HOST_USER', - default=settings.EMAIL_HOST_USER, - help_text=_( - 'Default: \'\' (Empty string). Username to use for the SMTP ' - 'server defined in EMAIL_HOST. If empty, Django won\'t attempt ' - 'authentication.' - ), -) -setting_django_email_port = namespace.add_setting( - global_name='EMAIL_PORT', - default=settings.EMAIL_PORT, - help_text=_( - 'Default: 25. Port to use for the SMTP server defined in EMAIL_HOST.' - ), -) -setting_django_email_timeout = namespace.add_setting( - global_name='EMAIL_TIMEOUT', - default=settings.EMAIL_TIMEOUT, - help_text=_( - 'Default: None. Specifies a timeout in seconds for blocking ' - 'operations like the connection attempt.' - ), -) -setting_django_email_user_tls = namespace.add_setting( - global_name='EMAIL_USE_TLS', - default=settings.EMAIL_USE_TLS, - help_text=_( - 'Default: False. Whether to use a TLS (secure) connection when ' - 'talking to the SMTP server. This is used for explicit TLS ' - 'connections, generally on port 587. If you are experiencing ' - 'hanging connections, see the implicit TLS setting EMAIL_USE_SSL.' - ), -) -setting_django_email_user_ssl = namespace.add_setting( - global_name='EMAIL_USE_SSL', - default=settings.EMAIL_USE_SSL, - help_text=_( - 'Default: False. Whether to use an implicit TLS (secure) connection ' - 'when talking to the SMTP server. In most email documentation this ' - 'type of TLS connection is referred to as SSL. It is generally used ' - 'on port 465. If you are experiencing problems, see the explicit ' - 'TLS setting EMAIL_USE_TLS. Note that EMAIL_USE_TLS/EMAIL_USE_SSL ' - 'are mutually exclusive, so only set one of those settings to True.' - ), -) -setting_django_file_upload_max_memory_size = namespace.add_setting( - global_name='FILE_UPLOAD_MAX_MEMORY_SIZE', - default=settings.FILE_UPLOAD_MAX_MEMORY_SIZE, - help_text=_( - 'Default: 2621440 (i.e. 2.5 MB). The maximum size (in bytes) ' - 'that an upload will be before it gets streamed to the file ' - 'system. See Managing files for details. See also ' - 'DATA_UPLOAD_MAX_MEMORY_SIZE.' - ), -) -setting_django_login_url = namespace.add_setting( - global_name='LOGIN_URL', - default=settings.LOGIN_URL, - help_text=_( - 'Default: \'/accounts/login/\' The URL where requests are ' - 'redirected for login, especially when using the login_required() ' - 'decorator. This setting also accepts named URL patterns which ' - 'can be used to reduce configuration duplication since you ' - 'don\'t have to define the URL in two places (settings ' - 'and URLconf).' - ) -) -setting_django_login_redirect_url = namespace.add_setting( - global_name='LOGIN_REDIRECT_URL', - default=settings.LOGIN_REDIRECT_URL, - help_text=_( - 'Default: \'/accounts/profile/\' The URL where requests are ' - 'redirected after login when the contrib.auth.login view gets no ' - 'next parameter. This is used by the login_required() decorator, ' - 'for example. This setting also accepts named URL patterns which ' - 'can be used to reduce configuration duplication since you don\'t ' - 'have to define the URL in two places (settings and URLconf).' - ), -) -setting_django_logout_redirect_url = namespace.add_setting( - global_name='LOGOUT_REDIRECT_URL', - default=settings.LOGOUT_REDIRECT_URL, - help_text=_( - 'Default: None. The URL where requests are redirected after a user ' - 'logs out using LogoutView (if the view doesn\'t get a next_page ' - 'argument). If None, no redirect will be performed and the logout ' - 'view will be rendered. This setting also accepts named URL ' - 'patterns which can be used to reduce configuration duplication ' - 'since you don\'t have to define the URL in two places (settings ' - 'and URLconf).' - ) -) -setting_django_internal_ips = namespace.add_setting( - global_name='INTERNAL_IPS', - default=settings.INTERNAL_IPS, - help_text=_( - 'A list of IP addresses, as strings, that: Allow the debug() ' - 'context processor to add some variables to the template context. ' - 'Can use the admindocs bookmarklets even if not logged in as a ' - 'staff user. Are marked as "internal" (as opposed to "EXTERNAL") ' - 'in AdminEmailHandler emails.' - ), -) -setting_django_languages = namespace.add_setting( - global_name='LANGUAGES', - default=settings.LANGUAGES, - help_text=_( - 'A list of all available languages. The list is a list of ' - 'two-tuples in the format (language code, language name) ' - 'for example, (\'ja\', \'Japanese\'). This specifies which ' - 'languages are available for language selection. ' - 'Generally, the default value should suffice. Only set this ' - 'setting if you want to restrict language selection to a ' - 'subset of the Django-provided languages. ' - ), -) -setting_django_language_code = namespace.add_setting( - global_name='LANGUAGE_CODE', - default=settings.LANGUAGE_CODE, - help_text=_( - 'A string representing the language code for this installation. ' - 'This should be in standard language ID format. For example, U.S. ' - 'English is "en-us". It serves two purposes: If the locale ' - 'middleware isn\'t in use, it decides which translation is served ' - 'to all users. If the locale middleware is active, it provides a ' - 'fallback language in case the user\'s preferred language can\'t ' - 'be determined or is not supported by the website. It also provides ' - 'the fallback translation when a translation for a given literal ' - 'doesn\'t exist for the user\'s preferred language.' - ), -) -setting_django_static_url = namespace.add_setting( - global_name='STATIC_URL', - default=settings.STATIC_URL, - help_text=_( - 'URL to use when referring to static files located in STATIC_ROOT. ' - 'Example: "/static/" or "http://static.example.com/" ' - 'If not None, this will be used as the base path for asset ' - 'definitions (the Media class) and the staticfiles app. ' - 'It must end in a slash if set to a non-empty value.' - ), -) -setting_django_staticfiles_storage = namespace.add_setting( - global_name='STATICFILES_STORAGE', - default=settings.STATICFILES_STORAGE, - help_text=_( - 'The file storage engine to use when collecting static files with ' - 'the collectstatic management command. A ready-to-use instance of ' - 'the storage backend defined in this setting can be found at ' - 'django.contrib.staticfiles.storage.staticfiles_storage.' - ), -) -setting_django_time_zone = namespace.add_setting( - global_name='TIME_ZONE', - default=settings.TIME_ZONE, - help_text=_( - 'A string representing the time zone for this installation. ' - 'Note that this isn\'t necessarily the time zone of the server. ' - 'For example, one server may serve multiple Django-powered sites, ' - 'each with a separate time zone setting.' - ), -) -setting_django_wsgi_application = namespace.add_setting( - global_name='WSGI_APPLICATION', - default=settings.WSGI_APPLICATION, - help_text=_( - 'The full Python path of the WSGI application object that Django\'s ' - 'built-in servers (e.g. runserver) will use. The django-admin ' - 'startproject management command will create a simple wsgi.py ' - 'file with an application callable in it, and point this setting ' - 'to that application.' - ), -) - -namespace = Namespace(label=_('Celery'), name='celery') - -setting_celery_broker_url = namespace.add_setting( - global_name='BROKER_URL', default=settings.BROKER_URL, - help_text=_( - 'Default: "amqp://". Default broker URL. This must be a URL in ' - 'the form of: transport://userid:password@hostname:port/virtual_host ' - 'Only the scheme part (transport://) is required, the rest is ' - 'optional, and defaults to the specific transports default values.' - ), -) -setting_celery_result_backend = namespace.add_setting( - global_name='CELERY_RESULT_BACKEND', - default=settings.CELERY_RESULT_BACKEND, - help_text=_( - 'Default: No result backend enabled by default. The backend used ' - 'to store task results (tombstones). Refer to ' - 'http://docs.celeryproject.org/en/v4.1.0/userguide/configuration.' - 'html#result-backend' - ) + default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')} ) diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index ded073d4e8..422b20682a 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,23 +1,11 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_shared_storage, setting_shared_storage_arguments ) -storage_sharedupload = import_string( +storage_sharedupload = get_storage_subclass( dotted_path=setting_shared_storage.value -)( - **yaml.load( - stream=setting_shared_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_shared_storage_arguments.value) diff --git a/mayan/apps/common/tests/base.py b/mayan/apps/common/tests/base.py index 5c08bffd57..3c352df9df 100644 --- a/mayan/apps/common/tests/base.py +++ b/mayan/apps/common/tests/base.py @@ -7,6 +7,7 @@ from django_downloadview import assert_download_response from mayan.apps.acls.tests.mixins import ACLTestCaseMixin from mayan.apps.permissions.classes import Permission from mayan.apps.smart_settings.classes import Namespace +from mayan.apps.user_management.tests.mixins import UserTestMixin from .mixins import ( ClientMethodsTestCaseMixin, ConnectionsCheckTestCaseMixin, @@ -21,7 +22,7 @@ class BaseTestCase( SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, RandomPrimaryKeyModelMonkeyPatchMixin, ACLTestCaseMixin, ModelTestCaseMixin, OpenFileCheckTestCaseMixin, - TempfileCheckTestCasekMixin, TestCase + TempfileCheckTestCasekMixin, UserTestMixin, TestCase ): """ This is the most basic test case class any test in the project should use. diff --git a/mayan/apps/common/tests/utils.py b/mayan/apps/common/tests/utils.py index 85daf0c524..5b7aac4993 100644 --- a/mayan/apps/common/tests/utils.py +++ b/mayan/apps/common/tests/utils.py @@ -1,6 +1,10 @@ +from __future__ import absolute_import, unicode_literals + from contextlib import contextmanager import sys +from django.utils.encoding import force_text + class NullFile(object): def write(self, string): @@ -13,3 +17,9 @@ def mute_stdout(): sys.stdout = NullFile() yield sys.stdout = stdout_old + + +def as_id_list(items): + return ','.join( + [force_text(item.pk) for item in items] + ) diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index 2d76f17d43..85447e2813 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -21,14 +21,6 @@ def check_for_sqlite(): return settings.DATABASES['default']['ENGINE'] == DJANGO_SQLITE_BACKEND and settings.DEBUG is False -def encapsulate(function): - # Workaround Django ticket 15791 - # Changeset 16045 - # http://stackoverflow.com/questions/6861601/ - # cannot-resolve-callable-context-variable/6955045#6955045 - return lambda: function - - def get_related_field(model, related_field_name): try: local_field_name, remaining_field_path = related_field_name.split( diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 672997f879..700ad2a381 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,11 +7,6 @@ import shutil from PIL import Image import PyPDF2 import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -20,16 +15,14 @@ from mayan.apps.storage.utils import NamedTemporaryFile from ..classes import ConverterBase from ..exceptions import PageCountError -from ..settings import setting_graphics_backend_config +from ..settings import setting_graphics_backend_arguments from ..literals import ( DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH ) -pdftoppm_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -).get( +pdftoppm_path = setting_graphics_backend_arguments.value.get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) @@ -39,26 +32,20 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + setting_graphics_backend_arguments.value.get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + setting_graphics_backend_arguments.value.get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) ) pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi) -pdfinfo_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -).get( +pdfinfo_path = setting_graphics_backend_arguments.value.get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 517b04ac51..cd6a99ce2b 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,15 +7,10 @@ import shutil from PIL import Image import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.mimetype.api import get_mimetype from mayan.apps.storage.settings import setting_temporary_directory from mayan.apps.storage.utils import ( @@ -27,16 +22,14 @@ from .literals import ( CONVERTER_OFFICE_FILE_MIMETYPES, DEFAULT_LIBREOFFICE_PATH, DEFAULT_PAGE_NUMBER, DEFAULT_PILLOW_FORMAT ) -from .settings import setting_graphics_backend_config +from .settings import setting_graphics_backend_arguments -logger = logging.getLogger(__name__) -BACKEND_CONFIG = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -) -libreoffice_path = BACKEND_CONFIG.get( +libreoffice_path = setting_graphics_backend_arguments.value.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH ) +logger = logging.getLogger(__name__) + class ConverterBase(object): def __init__(self, file_object, mime_type=None): @@ -62,9 +55,7 @@ class ConverterBase(object): pass def get_page(self, output_format=None): - output_format = output_format or yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + output_format = output_format or setting_graphics_backend_arguments.value.get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/forms.py b/mayan/apps/converter/forms.py index 73749d2897..3137f0dc80 100644 --- a/mayan/apps/converter/forms.py +++ b/mayan/apps/converter/forms.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + from .models import Transformation @@ -21,7 +18,7 @@ class TransformationForm(forms.ModelForm): def clean(self): try: - yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['arguments']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/converter/managers.py b/mayan/apps/converter/managers.py index 5856231203..45c4d02a10 100644 --- a/mayan/apps/converter/managers.py +++ b/mayan/apps/converter/managers.py @@ -2,16 +2,11 @@ from __future__ import unicode_literals import logging -import yaml - -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.contrib.contenttypes.models import ContentType from django.db import models, transaction +from mayan.apps.common.serialization import yaml_dump, yaml_load + from .transformations import BaseTransformation logger = logging.getLogger(__name__) @@ -23,8 +18,8 @@ class TransformationManager(models.Manager): self.create( content_type=content_type, object_id=obj.pk, - name=transformation.name, arguments=yaml.dump( - data=arguments, Dumper=SafeDumper + name=transformation.name, arguments=yaml_dump( + data=arguments ) ) @@ -96,9 +91,8 @@ class TransformationManager(models.Manager): # Some transformations don't require arguments # return an empty dictionary as ** doesn't allow None if transformation.arguments: - kwargs = yaml.load( + kwargs = yaml_load( stream=transformation.arguments, - Loader=SafeLoader ) else: kwargs = {} diff --git a/mayan/apps/converter/settings.py b/mayan/apps/converter/settings.py index 14ce07a295..0d30648fc2 100644 --- a/mayan/apps/converter/settings.py +++ b/mayan/apps/converter/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, @@ -16,22 +16,15 @@ setting_graphics_backend = namespace.add_setting( help_text=_('Graphics conversion backend to use.'), global_name='CONVERTER_GRAPHICS_BACKEND', ) -setting_graphics_backend_config = namespace.add_setting( - default=''' - {{ - libreoffice_path: {}, - pdftoppm_dpi: {}, - pdftoppm_format: {}, - pdftoppm_path: {}, - pdfinfo_path: {}, - pillow_format: {} - - }} - '''.replace('\n', '').format( - DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, - DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH, - DEFAULT_PILLOW_FORMAT - ), help_text=_( +setting_graphics_backend_arguments = namespace.add_setting( + default={ + 'libreoffice_path': DEFAULT_LIBREOFFICE_PATH, + 'pdftoppm_dpi': DEFAULT_PDFTOPPM_DPI, + 'pdftoppm_format': DEFAULT_PDFTOPPM_FORMAT, + 'pdftoppm_path': DEFAULT_PDFTOPPM_PATH, + 'pdfinfo_path': DEFAULT_PDFINFO_PATH, + 'pillow_format': DEFAULT_PILLOW_FORMAT, + }, help_text=_( 'Configuration options for the graphics conversion backend.' - ), global_name='CONVERTER_GRAPHICS_BACKEND_CONFIG', quoted=True + ), global_name='CONVERTER_GRAPHICS_BACKEND_ARGUMENTS' ) diff --git a/mayan/apps/converter/validators.py b/mayan/apps/converter/validators.py index 49c45d0cae..dd8562a945 100644 --- a/mayan/apps/converter/validators.py +++ b/mayan/apps/converter/validators.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + @deconstructible class YAMLValidator(object): @@ -20,7 +17,7 @@ class YAMLValidator(object): def __call__(self, value): value = value.strip() try: - yaml.load(stream=value, Loader=SafeLoader) + yaml_load(stream=value) except yaml.error.YAMLError: raise ValidationError( _('Enter a valid YAML value.'), diff --git a/mayan/apps/dependencies/management/commands/installjavascript.py b/mayan/apps/dependencies/management/commands/installdependencies.py similarity index 100% rename from mayan/apps/dependencies/management/commands/installjavascript.py rename to mayan/apps/dependencies/management/commands/installdependencies.py diff --git a/mayan/apps/django_gpg/settings.py b/mayan/apps/django_gpg/settings.py index b7c763f45f..651f54bcbc 100644 --- a/mayan/apps/django_gpg/settings.py +++ b/mayan/apps/django_gpg/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Signatures'), name='django_gpg') diff --git a/mayan/apps/document_indexing/tests/test_views.py b/mayan/apps/document_indexing/tests/test_views.py index dd554401d6..98f9f58b20 100644 --- a/mayan/apps/document_indexing/tests/test_views.py +++ b/mayan/apps/document_indexing/tests/test_views.py @@ -17,7 +17,11 @@ from .literals import ( from .mixins import IndexTestMixin, IndexViewTestMixin -class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase): +class IndexViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): + auto_upload_document = False + def test_index_create_view_no_permission(self): response = self._request_test_index_create_view() self.assertEqual(response.status_code, 403) @@ -75,6 +79,45 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT self.test_index.refresh_from_db() self.assertEqual(self.test_index.label, TEST_INDEX_LABEL_EDITED) + def test_index_rebuild_view_no_permission(self): + self.upload_document() + self._create_test_index() + + self.test_index.node_templates.create( + parent=self.test_index.template_root, + expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION, + link_documents=True + ) + + response = self._request_test_index_rebuild_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual(IndexInstanceNode.objects.count(), 0) + + def test_index_rebuild_view_with_access(self): + self.upload_document() + self._create_test_index() + + self.test_index.node_templates.create( + parent=self.test_index.template_root, + expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION, + link_documents=True + ) + + self.grant_access( + obj=self.test_index, + permission=permission_document_indexing_rebuild + ) + + response = self._request_test_index_rebuild_view() + self.assertEqual(response.status_code, 302) + + self.assertNotEqual(IndexInstanceNode.objects.count(), 0) + + +class IndexInstanceViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): def _request_index_instance_node_view(self, index_instance_node): return self.get( viewname='indexing:index_instance_node_view', kwargs={ @@ -103,9 +146,13 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT ) self.assertContains(response, text=TEST_INDEX_LABEL, status_code=200) + +class IndexToolsViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): def _request_indexes_rebuild_get_view(self): return self.get( - viewname='indexing:rebuild_index_instances', + viewname='indexing:rebuild_index_instances' ) def _request_indexes_rebuild_post_view(self): @@ -149,36 +196,3 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT # An instance root exists self.assertTrue(self.test_index.instance_root.pk) - - def test_index_rebuild_view_no_permission(self): - self._create_test_index() - - self.test_index.node_templates.create( - parent=self.test_index.template_root, - expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION, - link_documents=True - ) - - response = self._request_test_index_rebuild_view() - self.assertEqual(response.status_code, 404) - - self.assertEqual(IndexInstanceNode.objects.count(), 0) - - def test_index_rebuild_view_with_access(self): - self._create_test_index() - - self.test_index.node_templates.create( - parent=self.test_index.template_root, - expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION, - link_documents=True - ) - - self.grant_access( - obj=self.test_index, - permission=permission_document_indexing_rebuild - ) - - response = self._request_test_index_rebuild_view() - self.assertEqual(response.status_code, 302) - - self.assertNotEqual(IndexInstanceNode.objects.count(), 0) diff --git a/mayan/apps/document_indexing/views.py b/mayan/apps/document_indexing/views.py index 69082d67a6..384e45064f 100644 --- a/mayan/apps/document_indexing/views.py +++ b/mayan/apps/document_indexing/views.py @@ -308,7 +308,6 @@ class IndexListView(SingleObjectListView): def get_extra_context(self): return { - 'hide_object': True, 'hide_links': True, 'hide_object': True, 'no_results_icon': icon_index, diff --git a/mayan/apps/document_parsing/settings.py b/mayan/apps/document_parsing/settings.py index c6626fbbd1..7bf2b78d3a 100644 --- a/mayan/apps/document_parsing/settings.py +++ b/mayan/apps/document_parsing/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document parsing'), name='document_parsing') diff --git a/mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py b/mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py new file mode 100644 index 0000000000..b77b3ae66e --- /dev/null +++ b/mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-07-11 05:44 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.document_signatures.models +import mayan.apps.storage.classes + + +class Migration(migrations.Migration): + + dependencies = [ + ('document_signatures', '0008_auto_20180429_0759'), + ] + + operations = [ + migrations.AlterField( + model_name='detachedsignature', + name='signature_file', + field=models.FileField(blank=True, null=True, storage=mayan.apps.storage.classes.FakeStorageSubclass(), upload_to=mayan.apps.document_signatures.models.upload_to, verbose_name='Signature file'), + ), + ] diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index 6e0b75ec23..7a5e9dcfe1 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document signatures'), name='signatures') @@ -18,9 +18,9 @@ setting_storage_backend = namespace.add_setting( ) setting_storage_backend_arguments = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_signatures') - ), quoted=True, help_text=_( + default={ + 'location': os.path.join(settings.MEDIA_ROOT, 'document_signatures') + }, help_text=_( 'Arguments to pass to the SIGNATURE_STORAGE_BACKEND. ' ) ) diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 45d5d63d5e..423f11c1e5 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,23 +1,11 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_storage_backend, setting_storage_backend_arguments ) -storage_detachedsignature = import_string( +storage_detachedsignature = get_storage_subclass( dotted_path=setting_storage_backend.value -)( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_storage_backend_arguments.value) diff --git a/mayan/apps/document_states/api_views.py b/mayan/apps/document_states/api_views.py index 5f998db115..90df8863cb 100644 --- a/mayan/apps/document_states/api_views.py +++ b/mayan/apps/document_states/api_views.py @@ -31,7 +31,7 @@ from .storages import storage_workflowimagecache from .tasks import task_generate_workflow_image -class APIDocumentTypeWorkflowListView(generics.ListAPIView): +class APIDocumentTypeWorkflowRuntimeProxyListView(generics.ListAPIView): """ get: Returns a list of all the document type workflows. """ @@ -214,7 +214,7 @@ class APIWorkflowImageView(generics.RetrieveAPIView): return response -class APIWorkflowListView(generics.ListCreateAPIView): +class APIWorkflowRuntimeProxyListView(generics.ListCreateAPIView): """ get: Returns a list of all the workflows. post: Create a new workflow. @@ -229,7 +229,7 @@ class APIWorkflowListView(generics.ListCreateAPIView): if not self.request: return None - return super(APIWorkflowListView, self).get_serializer(*args, **kwargs) + return super(APIWorkflowRuntimeProxyListView, self).get_serializer(*args, **kwargs) def get_serializer_class(self): if self.request.method == 'GET': diff --git a/mayan/apps/document_states/apps.py b/mayan/apps/document_states/apps.py index dc048bb3a0..387aefe40d 100644 --- a/mayan/apps/document_states/apps.py +++ b/mayan/apps/document_states/apps.py @@ -29,27 +29,27 @@ from .handlers import ( ) from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events from .links import ( - link_document_workflow_instance_list, link_setup_document_type_workflows, - link_setup_workflow_document_types, link_setup_workflow_create, - link_setup_workflow_delete, link_setup_workflow_edit, - link_setup_workflow_list, link_setup_workflow_states, - link_setup_workflow_state_action_delete, - link_setup_workflow_state_action_edit, - link_setup_workflow_state_action_list, - link_setup_workflow_state_action_selection, - link_setup_workflow_state_create, link_setup_workflow_state_delete, - link_setup_workflow_state_edit, link_setup_workflow_transitions, - link_setup_workflow_transition_create, - link_setup_workflow_transition_delete, link_setup_workflow_transition_edit, - link_setup_workflow_transition_field_create, - link_setup_workflow_transition_field_delete, - link_setup_workflow_transition_field_edit, - link_setup_workflow_transition_field_list, - link_tool_launch_all_workflows, link_workflow_instance_detail, + link_workflow_instance_list, link_document_type_workflow_templates, + link_workflow_template_document_types, link_workflow_template_create, + link_workflow_template_delete, link_workflow_template_edit, + link_workflow_template_list, link_workflow_template_state_list, + link_workflow_template_state_action_delete, + link_workflow_template_state_action_edit, + link_workflow_template_state_action_list, + link_workflow_template_state_action_selection, + link_workflow_template_state_create, link_workflow_template_state_delete, + link_workflow_template_state_edit, link_workflow_template_transition_list, + link_workflow_template_transition_create, + link_workflow_template_transition_delete, link_workflow_template_transition_edit, + link_workflow_template_transition_field_create, + link_workflow_template_transition_field_delete, + link_workflow_template_transition_field_edit, + link_workflow_template_transition_field_list, + link_tool_launch_workflows, link_workflow_instance_detail, link_workflow_instance_transition, link_workflow_runtime_proxy_document_list, - link_workflow_runtime_proxy_list, link_workflow_preview, + link_workflow_runtime_proxy_list, link_workflow_template_preview, link_workflow_runtime_proxy_state_document_list, link_workflow_runtime_proxy_state_list, - link_workflow_transition_events + link_workflow_template_transition_events ) from .permissions import ( permission_workflow_delete, permission_workflow_edit, @@ -319,49 +319,49 @@ class DocumentStatesApp(MayanAppConfig): ) menu_facet.bind_links( - links=(link_document_workflow_instance_list,), sources=(Document,) + links=(link_workflow_instance_list,), sources=(Document,) ) menu_list_facet.bind_links( links=( link_acl_list, link_events_for_object, link_object_event_types_user_subcriptions_list, - link_setup_workflow_document_types, - link_setup_workflow_states, link_setup_workflow_transitions, - link_workflow_preview + link_workflow_template_document_types, + link_workflow_template_state_list, link_workflow_template_transition_list, + link_workflow_template_preview ), sources=(Workflow,) ) menu_list_facet.bind_links( links=( - link_setup_document_type_workflows, + link_document_type_workflow_templates, ), sources=(DocumentType,) ) menu_main.bind_links(links=(link_workflow_runtime_proxy_list,), position=10) menu_object.bind_links( links=( - link_setup_workflow_delete, link_setup_workflow_edit + link_workflow_template_delete, link_workflow_template_edit ), sources=(Workflow,) ) menu_object.bind_links( links=( - link_setup_workflow_state_edit, - link_setup_workflow_state_action_list, - link_setup_workflow_state_delete + link_workflow_template_state_edit, + link_workflow_template_state_action_list, + link_workflow_template_state_delete ), sources=(WorkflowState,) ) menu_object.bind_links( links=( - link_setup_workflow_transition_edit, - link_workflow_transition_events, - link_setup_workflow_transition_field_list, link_acl_list, - link_setup_workflow_transition_delete + link_workflow_template_transition_edit, + link_workflow_template_transition_events, + link_workflow_template_transition_field_list, link_acl_list, + link_workflow_template_transition_delete ), sources=(WorkflowTransition,) ) menu_object.bind_links( links=( - link_setup_workflow_transition_field_delete, - link_setup_workflow_transition_field_edit + link_workflow_template_transition_field_delete, + link_workflow_template_transition_field_edit ), sources=(WorkflowTransitionField,) ) menu_object.bind_links( @@ -384,21 +384,21 @@ class DocumentStatesApp(MayanAppConfig): ) menu_object.bind_links( links=( - link_setup_workflow_state_action_edit, + link_workflow_template_state_action_edit, link_object_error_list, - link_setup_workflow_state_action_delete, + link_workflow_template_state_action_delete, ), sources=(WorkflowStateAction,) ) menu_secondary.bind_links( - links=(link_setup_workflow_list, link_setup_workflow_create), + links=(link_workflow_template_list, link_workflow_template_create), sources=( - Workflow, 'document_states:setup_workflow_create', - 'document_states:setup_workflow_list' + Workflow, 'document_states:workflow_template_create', + 'document_states:workflow_template_list' ) ) menu_secondary.bind_links( - links=(link_setup_workflow_transition_field_create,), + links=(link_workflow_template_transition_field_create,), sources=( WorkflowTransition, ) @@ -410,31 +410,31 @@ class DocumentStatesApp(MayanAppConfig): ) ) menu_secondary.bind_links( - links=(link_setup_workflow_state_action_selection,), + links=(link_workflow_template_state_action_selection,), sources=( WorkflowState, ) ) menu_secondary.bind_links( links=( - link_setup_workflow_transition_create, + link_workflow_template_transition_create, ), sources=( WorkflowTransition, - 'document_states:setup_workflow_transition_list', + 'document_states:workflow_template_transition_list', ) ) menu_secondary.bind_links( links=( - link_setup_workflow_state_create, + link_workflow_template_state_create, ), sources=( WorkflowState, - 'document_states:setup_workflow_state_list', + 'document_states:workflow_template_state_list', ) ) - menu_setup.bind_links(links=(link_setup_workflow_list,)) + menu_setup.bind_links(links=(link_workflow_template_list,)) - menu_tools.bind_links(links=(link_tool_launch_all_workflows,)) + menu_tools.bind_links(links=(link_tool_launch_workflows,)) post_save.connect( dispatch_uid='workflows_handler_launch_workflow', diff --git a/mayan/apps/document_states/html_widgets.py b/mayan/apps/document_states/html_widgets.py index 98ad2d817f..58d3da1773 100644 --- a/mayan/apps/document_states/html_widgets.py +++ b/mayan/apps/document_states/html_widgets.py @@ -1,9 +1,7 @@ from __future__ import unicode_literals -from django import forms from django.template.loader import render_to_string -from django.urls import reverse -from django.utils.html import format_html_join, mark_safe +from django.utils.html import format_html_join def widget_transition_events(transition): diff --git a/mayan/apps/document_states/icons.py b/mayan/apps/document_states/icons.py index 9fa519fac1..3115901935 100644 --- a/mayan/apps/document_states/icons.py +++ b/mayan/apps/document_states/icons.py @@ -1,45 +1,44 @@ from __future__ import absolute_import, unicode_literals from mayan.apps.appearance.classes import Icon -from mayan.apps.documents.icons import icon_document_type +from mayan.apps.documents.icons import icon_document, icon_document_type icon_workflow = Icon(driver_name='fontawesome', symbol='sitemap') -icon_document_type_workflow_list = icon_workflow +icon_tool_launch_workflows = icon_workflow -icon_document_workflow_instance_list = Icon( - driver_name='fontawesome', symbol='sitemap' -) -icon_setup_workflow_list = Icon(driver_name='fontawesome', symbol='sitemap') -icon_tool_launch_all_workflows = Icon( - driver_name='fontawesome', symbol='sitemap' -) -icon_workflow_create = Icon( +icon_document_type_workflow_list = icon_workflow +icon_workflow_template_create = Icon( driver_name='fontawesome-dual', primary_symbol='sitemap', secondary_symbol='plus' ) -icon_workflow_delete = Icon(driver_name='fontawesome', symbol='times') -icon_workflow_document_type_list = icon_document_type -icon_workflow_edit = Icon(driver_name='fontawesome', symbol='pencil-alt') -icon_workflow_list = Icon(driver_name='fontawesome', symbol='sitemap') -icon_workflow_preview = Icon(driver_name='fontawesome', symbol='eye') - -icon_workflow_instance_detail = Icon( - driver_name='fontawesome', symbol='sitemap' +icon_workflow_template_delete = Icon(driver_name='fontawesome', symbol='times') +icon_workflow_template_document_type_list = icon_document_type +icon_workflow_template_edit = Icon( + driver_name='fontawesome', symbol='pencil-alt' ) +icon_workflow_template_list = icon_workflow +icon_workflow_template_preview = Icon(driver_name='fontawesome', symbol='eye') + +# Workflow instances + +icon_workflow_instance_detail = icon_workflow +icon_workflow_instance_list = icon_workflow icon_workflow_instance_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) -icon_workflow_runtime_proxy_document_list = icon_document_type -icon_workflow_runtime_proxy_list = Icon( - driver_name='fontawesome', symbol='sitemap' -) -icon_workflow_runtime_proxy_state_document_list = icon_document_type +# Workflow runtime proxies + +icon_workflow_runtime_proxy_document_list = icon_document +icon_workflow_runtime_proxy_list = icon_workflow +icon_workflow_runtime_proxy_state_document_list = icon_document icon_workflow_runtime_proxy_state_list = Icon( driver_name='fontawesome', symbol='circle' ) +# Workflow transition states + icon_workflow_state_action_delete = Icon( driver_name='fontawesome', symbol='times' ) @@ -57,6 +56,8 @@ icon_workflow_state_create = Icon( icon_workflow_state_delete = Icon(driver_name='fontawesome', symbol='times') icon_workflow_state_edit = Icon(driver_name='fontawesome', symbol='pencil-alt') +# Workflow transition state actions + icon_workflow_state_action = Icon(driver_name='fontawesome', symbol='code') icon_workflow_state_action_delete = Icon( driver_name='fontawesome', symbol='times' @@ -71,6 +72,9 @@ icon_workflow_state_action_selection = Icon( icon_workflow_state_action_list = Icon( driver_name='fontawesome', symbol='code' ) + +# Workflow transitions + icon_workflow_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) @@ -85,7 +89,11 @@ icon_workflow_transition_edit = Icon( driver_name='fontawesome', symbol='pencil-alt' ) -icon_workflow_transition_field = Icon(driver_name='fontawesome', symbol='table') +# Workflow transition fields + +icon_workflow_transition_field = Icon( + driver_name='fontawesome', symbol='table' +) icon_workflow_transition_field_delete = Icon( driver_name='fontawesome', symbol='times' ) @@ -99,7 +107,6 @@ icon_workflow_transition_field_create = Icon( icon_workflow_transition_field_list = Icon( driver_name='fontawesome', symbol='table' ) - icon_workflow_transition_triggers = Icon( driver_name='fontawesome', symbol='bolt' ) diff --git a/mayan/apps/document_states/links.py b/mayan/apps/document_states/links.py index 1404ff2644..08e8d001cd 100644 --- a/mayan/apps/document_states/links.py +++ b/mayan/apps/document_states/links.py @@ -11,179 +11,185 @@ from .permissions import ( permission_workflow_view, ) -link_setup_document_type_workflows = Link( +# Workflow templates + +link_document_type_workflow_templates = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_document_type_workflow_list', permissions=(permission_document_type_edit,), text=_('Workflows'), - view='document_states:document_type_workflows', + view='document_states:document_type_workflow_templates', ) -link_setup_workflow_create = Link( - icon_class_path='mayan.apps.document_states.icons.icon_workflow_create', +link_workflow_template_create = Link( + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_create', permissions=(permission_workflow_create,), - text=_('Create workflow'), view='document_states:setup_workflow_create' + text=_('Create workflow'), view='document_states:workflow_template_create' ) -link_setup_workflow_delete = Link( +link_workflow_template_delete = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_delete', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_delete', permissions=(permission_workflow_delete,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_delete', + view='document_states:workflow_template_delete', ) -link_setup_workflow_document_types = Link( +link_workflow_template_document_types = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_document_type_list', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_document_type_list', permissions=(permission_workflow_edit,), text=_('Document types'), - view='document_states:setup_workflow_document_types', + view='document_states:workflow_template_document_types', ) -link_setup_workflow_edit = Link( +link_workflow_template_edit = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_edit', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_edit', + text=_('Edit'), view='document_states:workflow_template_edit', ) -link_setup_workflow_list = Link( - icon_class_path='mayan.apps.document_states.icons.icon_setup_workflow_list', +link_workflow_template_list = Link( + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_list', permissions=(permission_workflow_view,), text=_('Workflows'), - view='document_states:setup_workflow_list' + view='document_states:workflow_template_list' ) -link_setup_workflow_state_action_delete = Link( +link_workflow_template_preview = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_preview', + permissions=(permission_workflow_view,), + text=_('Preview'), view='document_states:workflow_template_preview' +) + +# Workflow template state actions + +link_workflow_template_state_action_delete = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_state_action_delete', + view='document_states:workflow_template_state_action_delete', ) -link_setup_workflow_state_action_edit = Link( +link_workflow_template_state_action_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_state_action_edit', + text=_('Edit'), view='document_states:workflow_template_state_action_edit', ) -link_setup_workflow_state_action_list = Link( +link_workflow_template_state_action_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action_list', permissions=(permission_workflow_edit,), text=_('Actions'), - view='document_states:setup_workflow_state_action_list', + view='document_states:workflow_template_state_action_list', ) -link_setup_workflow_state_action_selection = Link( +link_workflow_template_state_action_selection = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action', permissions=(permission_workflow_edit,), text=_('Create action'), - view='document_states:setup_workflow_state_action_selection', + view='document_states:workflow_template_state_action_selection', ) -link_setup_workflow_state_create = Link( + +# Workflow template states + +link_workflow_template_state_create = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_create', permissions=(permission_workflow_edit,), text=_('Create state'), - view='document_states:setup_workflow_state_create', + view='document_states:workflow_template_state_create', ) -link_setup_workflow_state_delete = Link( +link_workflow_template_state_delete = Link( args='object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_state_delete', + view='document_states:workflow_template_state_delete', ) -link_setup_workflow_state_edit = Link( +link_workflow_template_state_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_state_edit', + text=_('Edit'), view='document_states:workflow_template_state_edit', ) -link_setup_workflow_states = Link( +link_workflow_template_state_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state', permissions=(permission_workflow_view,), text=_('States'), - view='document_states:setup_workflow_state_list', + view='document_states:workflow_template_state_list', ) -link_setup_workflow_transition_create = Link( + +# Workflow template transitions + +link_workflow_template_transition_create = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_create', permissions=(permission_workflow_edit,), text=_('Create transition'), - view='document_states:setup_workflow_transition_create', + view='document_states:workflow_template_transition_create', ) -link_setup_workflow_transition_delete = Link( +link_workflow_template_transition_delete = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_transition_delete', + view='document_states:workflow_template_transition_delete', ) -link_setup_workflow_transition_edit = Link( +link_workflow_template_transition_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_transition_edit', + text=_('Edit'), view='document_states:workflow_template_transition_edit', ) -link_setup_workflow_transitions = Link( - args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition', - permissions=(permission_workflow_view,), text=_('Transitions'), - view='document_states:setup_workflow_transition_list', -) -link_workflow_transition_events = Link( +link_workflow_template_transition_events = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_triggers', permissions=(permission_workflow_edit,), text=_('Transition triggers'), - view='document_states:setup_workflow_transition_events' + view='document_states:workflow_template_transition_events' +) +link_workflow_template_transition_list = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition', + permissions=(permission_workflow_view,), text=_('Transitions'), + view='document_states:workflow_template_transition_list', ) # Workflow transition fields -link_setup_workflow_transition_field_create = Link( + +link_workflow_template_transition_field_create = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field', permissions=(permission_workflow_edit,), text=_('Create field'), - view='document_states:setup_workflow_transition_field_create', + view='document_states:workflow_template_transition_field_create', ) -link_setup_workflow_transition_field_delete = Link( +link_workflow_template_transition_field_delete = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_transition_field_delete', + view='document_states:workflow_template_transition_field_delete', ) -link_setup_workflow_transition_field_edit = Link( +link_workflow_template_transition_field_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_transition_field_edit', + text=_('Edit'), view='document_states:workflow_template_transition_field_edit', ) -link_setup_workflow_transition_field_list = Link( +link_workflow_template_transition_field_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_list', permissions=(permission_workflow_edit,), text=_('Fields'), - view='document_states:setup_workflow_transition_field_list', -) - -link_workflow_preview = Link( - args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_preview', - permissions=(permission_workflow_view,), - text=_('Preview'), view='document_states:workflow_preview' -) -link_tool_launch_all_workflows = Link( - icon_class_path='mayan.apps.document_states.icons.icon_tool_launch_all_workflows', - permissions=(permission_workflow_tools,), - text=_('Launch all workflows'), - view='document_states:tool_launch_all_workflows' + view='document_states:workflow_template_transition_field_list', ) # Document workflow instances -link_document_workflow_instance_list = Link( - args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_document_workflow_instance_list', - permissions=(permission_workflow_view,), text=_('Workflows'), - view='document_states:document_workflow_instance_list', -) + link_workflow_instance_detail = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_detail', permissions=(permission_workflow_view,), text=_('Detail'), view='document_states:workflow_instance_detail', ) +link_workflow_instance_list = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_list', + permissions=(permission_workflow_view,), text=_('Workflows'), + view='document_states:workflow_instance_list', +) link_workflow_instance_transition = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_transition', @@ -192,28 +198,38 @@ link_workflow_instance_transition = Link( ) # Runtime proxies + link_workflow_runtime_proxy_document_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_document_list', permissions=(permission_workflow_view,), text=_('Workflow documents'), - view='document_states:workflow_document_list', + view='document_states:workflow_runtime_proxy_document_list', ) link_workflow_runtime_proxy_list = Link( icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_list', permissions=(permission_workflow_view,), - text=_('Workflows'), view='document_states:workflow_list' + text=_('Workflows'), view='document_states:workflow_runtime_proxy_list' ) link_workflow_runtime_proxy_state_document_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_state_document_list', permissions=(permission_workflow_view,), text=_('State documents'), - view='document_states:workflow_state_document_list', + view='document_states:workflow_runtime_proxy_state_document_list', ) link_workflow_runtime_proxy_state_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_state_list', permissions=(permission_workflow_view,), - text=_('States'), view='document_states:workflow_state_list', + text=_('States'), view='document_states:workflow_runtime_proxy_state_list', +) + +# Tools + +link_tool_launch_workflows = Link( + icon_class_path='mayan.apps.document_states.icons.icon_tool_launch_workflows', + permissions=(permission_workflow_tools,), + text=_('Launch all workflows'), + view='document_states:tool_launch_workflows' ) diff --git a/mayan/apps/document_states/settings.py b/mayan/apps/document_states/settings.py index b90ae5e382..26298fbdf9 100644 --- a/mayan/apps/document_states/settings.py +++ b/mayan/apps/document_states/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Workflows'), name='document_states') diff --git a/mayan/apps/document_states/storages.py b/mayan/apps/document_states/storages.py index 8a689634c7..d4ab8de524 100644 --- a/mayan/apps/document_states/storages.py +++ b/mayan/apps/document_states/storages.py @@ -1,12 +1,12 @@ from __future__ import unicode_literals -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_workflowimagecache_storage, setting_workflowimagecache_storage_arguments ) -storage_workflowimagecache = import_string( +storage_workflowimagecache = get_storage_subclass( dotted_path=setting_workflowimagecache_storage.value )(**setting_workflowimagecache_storage_arguments.value) diff --git a/mayan/apps/document_states/tests/literals.py b/mayan/apps/document_states/tests/literals.py index c17a870e9c..63d758efc3 100644 --- a/mayan/apps/document_states/tests/literals.py +++ b/mayan/apps/document_states/tests/literals.py @@ -24,4 +24,3 @@ TEST_WORKFLOW_TRANSITION_LABEL_EDITED = 'test transition label edited' TEST_INDEX_TEMPLATE_METADATA_EXPRESSION = '{{{{ document.workflow.{}.get_current_state }}}}'.format( TEST_WORKFLOW_INTERNAL_NAME ) - diff --git a/mayan/apps/document_states/tests/mixins.py b/mayan/apps/document_states/tests/mixins.py index f8ea985a1c..ff09844425 100644 --- a/mayan/apps/document_states/tests/mixins.py +++ b/mayan/apps/document_states/tests/mixins.py @@ -38,19 +38,19 @@ class WorkflowStateViewTestMixin(object): data.update(extra_data) return self.post( - viewname='document_states:setup_workflow_state_create', + viewname='document_states:workflow_template_state_create', kwargs={'pk': self.test_workflow.pk}, data=data ) def _request_test_workflow_state_delete_view(self): return self.post( - viewname='document_states:setup_workflow_state_delete', + viewname='document_states:workflow_template_state_delete', kwargs={'pk': self.test_workflow_state_1.pk} ) def _request_test_workflow_state_edit_view(self): return self.post( - viewname='document_states:setup_workflow_state_edit', + viewname='document_states:workflow_template_state_edit', kwargs={'pk': self.test_workflow_state_1.pk}, data={ 'label': TEST_WORKFLOW_STATE_LABEL_EDITED } @@ -58,7 +58,7 @@ class WorkflowStateViewTestMixin(object): def _request_test_workflow_state_list_view(self): return self.get( - viewname='document_states:setup_workflow_state_list', + viewname='document_states:workflow_template_state_list', kwargs={'pk': self.test_workflow.pk} ) @@ -120,7 +120,7 @@ class WorkflowTestMixin(object): class WorkflowTransitionViewTestMixin(object): def _request_test_workflow_transition_create_view(self): return self.post( - viewname='document_states:setup_workflow_transition_create', + viewname='document_states:workflow_template_transition_create', kwargs={'pk': self.test_workflow.pk}, data={ 'label': TEST_WORKFLOW_TRANSITION_LABEL, 'origin_state': self.test_workflow_state_1.pk, @@ -130,13 +130,13 @@ class WorkflowTransitionViewTestMixin(object): def _request_test_workflow_transition_delete_view(self): return self.post( - viewname='document_states:setup_workflow_transition_delete', + viewname='document_states:workflow_template_transition_delete', kwargs={'pk': self.test_workflow_transition.pk} ) def _request_test_workflow_transition_edit_view(self): return self.post( - viewname='document_states:setup_workflow_transition_edit', + viewname='document_states:workflow_template_transition_edit', kwargs={'pk': self.test_workflow_transition.pk}, data={ 'label': TEST_WORKFLOW_TRANSITION_LABEL_EDITED, 'origin_state': self.test_workflow_state_1.pk, @@ -146,7 +146,7 @@ class WorkflowTransitionViewTestMixin(object): def _request_test_workflow_transition_list_view(self): return self.get( - viewname='document_states:setup_workflow_transition_list', + viewname='document_states:workflow_template_transition_list', kwargs={'pk': self.test_workflow.pk} ) @@ -163,7 +163,7 @@ class WorkflowTransitionViewTestMixin(object): class WorkflowViewTestMixin(object): def _request_test_workflow_create_view(self): return self.post( - viewname='document_states:setup_workflow_create', data={ + viewname='document_states:workflow_template_create', data={ 'label': TEST_WORKFLOW_LABEL, 'internal_name': TEST_WORKFLOW_INTERNAL_NAME, } @@ -171,14 +171,14 @@ class WorkflowViewTestMixin(object): def _request_test_workflow_delete_view(self): return self.post( - viewname='document_states:setup_workflow_delete', kwargs={ + viewname='document_states:workflow_template_delete', kwargs={ 'pk': self.test_workflow.pk } ) def _request_test_workflow_edit_view(self): return self.post( - viewname='document_states:setup_workflow_edit', kwargs={ + viewname='document_states:workflow_template_edit', kwargs={ 'pk': self.test_workflow.pk, }, data={ 'label': TEST_WORKFLOW_LABEL_EDITED, @@ -188,12 +188,12 @@ class WorkflowViewTestMixin(object): def _request_test_workflow_list_view(self): return self.get( - viewname='document_states:setup_workflow_list', + viewname='document_states:workflow_template_list', ) - def _request_test_workflow_preview_view(self): + def _request_test_workflow_template_preview_view(self): return self.get( - viewname='document_states:workflow_preview', kwargs={ + viewname='document_states:workflow_template_preview', kwargs={ 'pk': self.test_workflow.pk, } ) diff --git a/mayan/apps/document_states/tests/test_workflow_state_action_views.py b/mayan/apps/document_states/tests/test_workflow_state_action_views.py index e79853fd90..c1b54093a8 100644 --- a/mayan/apps/document_states/tests/test_workflow_state_action_views.py +++ b/mayan/apps/document_states/tests/test_workflow_state_action_views.py @@ -15,7 +15,7 @@ class WorkflowStateActionViewTestCase(WorkflowStateActionTestMixin, WorkflowTest def _request_test_document_state_action_view(self): return self.get( - viewname='document_states:setup_workflow_state_action_list', + viewname='document_states:workflow_template_state_action_list', kwargs={'pk': self.test_workflow_state.pk} ) diff --git a/mayan/apps/document_states/tests/test_workflow_transition_views.py b/mayan/apps/document_states/tests/test_workflow_transition_views.py index c298e5dde4..c4a004871c 100644 --- a/mayan/apps/document_states/tests/test_workflow_transition_views.py +++ b/mayan/apps/document_states/tests/test_workflow_transition_views.py @@ -212,7 +212,7 @@ class WorkflowTransitionEventViewTestCase( ): def _request_test_workflow_transition_event_list_view(self): return self.get( - viewname='document_states:setup_workflow_transition_events', + viewname='document_states:workflow_template_transition_events', kwargs={'pk': self.test_workflow_transition.pk} ) @@ -256,7 +256,7 @@ class WorkflowTransitionFieldViewTestCase( def _request_test_workflow_transition_field_list_view(self): return self.get( - viewname='document_states:setup_workflow_transition_field_list', + viewname='document_states:workflow_template_transition_field_list', kwargs={'pk': self.test_workflow_transition.pk} ) @@ -286,7 +286,7 @@ class WorkflowTransitionFieldViewTestCase( def _request_workflow_transition_field_create_view(self): return self.post( - viewname='document_states:setup_workflow_transition_field_create', + viewname='document_states:workflow_template_transition_field_create', kwargs={'pk': self.test_workflow_transition.pk}, data={ 'field_type': TEST_WORKFLOW_TRANSITION_FIELD_TYPE, @@ -324,7 +324,7 @@ class WorkflowTransitionFieldViewTestCase( def _request_workflow_transition_field_delete_view(self): return self.post( - viewname='document_states:setup_workflow_transition_field_delete', + viewname='document_states:workflow_template_transition_field_delete', kwargs={'pk': self.test_workflow_transition_field.pk}, ) diff --git a/mayan/apps/document_states/tests/test_workflow_views.py b/mayan/apps/document_states/tests/test_workflow_views.py index 24df83a095..1e738d7d53 100644 --- a/mayan/apps/document_states/tests/test_workflow_views.py +++ b/mayan/apps/document_states/tests/test_workflow_views.py @@ -93,31 +93,31 @@ class WorkflowViewTestCase( self.assertEqual(response.status_code, 200) self.assertContains(response, text=self.test_workflow.label) - def test_workflow_preview_view_no_access(self): + def test_workflow_template_preview_view_no_access(self): self._create_test_workflow() - response = self._request_test_workflow_preview_view() + response = self._request_test_workflow_template_preview_view() self.assertEqual(response.status_code, 404) self.assertTrue(self.test_workflow in Workflow.objects.all()) - def test_workflow_preview_view_with_access(self): + def test_workflow_template_preview_view_with_access(self): self._create_test_workflow() self.grant_access( obj=self.test_workflow, permission=permission_workflow_view ) - response = self._request_test_workflow_preview_view() + response = self._request_test_workflow_template_preview_view() self.assertEqual(response.status_code, 200) class WorkflowToolViewTestCase(WorkflowTestMixin, GenericDocumentViewTestCase): def _request_workflow_launch_view(self): return self.post( - viewname='document_states:tool_launch_all_workflows', + viewname='document_states:tool_launch_workflows', ) - def test_tool_launch_all_workflows_view_no_permission(self): + def test_tool_launch_workflows_view_no_permission(self): self._create_test_workflow(add_document_type=True) self._create_test_workflow_states() self._create_test_workflow_transition() @@ -129,7 +129,7 @@ class WorkflowToolViewTestCase(WorkflowTestMixin, GenericDocumentViewTestCase): self.assertEqual(self.test_document.workflows.count(), 0) - def test_tool_launch_all_workflows_view_with_permission(self): + def test_tool_launch_workflows_view_with_permission(self): self._create_test_workflow(add_document_type=True) self._create_test_workflow_states() self._create_test_workflow_transition() diff --git a/mayan/apps/document_states/urls.py b/mayan/apps/document_states/urls.py index ebc3e6acd9..6b36dd490c 100644 --- a/mayan/apps/document_states/urls.py +++ b/mayan/apps/document_states/urls.py @@ -3,222 +3,247 @@ from __future__ import unicode_literals from django.conf.urls import url from .api_views import ( - APIDocumentTypeWorkflowListView, APIWorkflowDocumentTypeList, + APIDocumentTypeWorkflowRuntimeProxyListView, APIWorkflowDocumentTypeList, APIWorkflowDocumentTypeView, APIWorkflowImageView, APIWorkflowInstanceListView, APIWorkflowInstanceView, - APIWorkflowInstanceLogEntryListView, APIWorkflowListView, + APIWorkflowInstanceLogEntryListView, APIWorkflowRuntimeProxyListView, APIWorkflowStateListView, APIWorkflowStateView, APIWorkflowTransitionListView, APIWorkflowTransitionView, APIWorkflowView ) -from .views import ( - DocumentWorkflowInstanceListView, SetupWorkflowCreateView, - SetupWorkflowDeleteView, SetupWorkflowDocumentTypesView, - SetupWorkflowEditView, SetupWorkflowListView, - SetupWorkflowStateActionCreateView, SetupWorkflowStateActionDeleteView, - SetupWorkflowStateActionEditView, SetupWorkflowStateActionListView, - SetupWorkflowStateActionSelectionView, SetupWorkflowStateCreateView, - SetupWorkflowStateDeleteView, SetupWorkflowStateEditView, - SetupWorkflowStateListView, SetupWorkflowTransitionListView, - SetupWorkflowTransitionCreateView, SetupWorkflowTransitionDeleteView, - SetupWorkflowTransitionEditView, - SetupWorkflowTransitionTriggerEventListView, ToolLaunchAllWorkflows, - WorkflowDocumentListView, WorkflowInstanceDetailView, - WorkflowInstanceTransitionExecuteView, WorkflowInstanceTransitionSelectView, - WorkflowListView, WorkflowPreviewView, WorkflowStateDocumentListView, - WorkflowStateListView, +from .views.workflow_instance_views import ( + WorkflowInstanceDetailView, WorkflowInstanceListView, + WorkflowInstanceTransitionSelectView, + WorkflowInstanceTransitionExecuteView ) -from .views.workflow_views import ( - SetupDocumentTypeWorkflowsView, SetupWorkflowTransitionFieldCreateView, - SetupWorkflowTransitionFieldDeleteView, - SetupWorkflowTransitionFieldEditView, SetupWorkflowTransitionFieldListView +from .views.workflow_proxy_views import ( + WorkflowRuntimeProxyDocumentListView, + WorkflowRuntimeProxyListView, WorkflowRuntimeProxyStateDocumentListView, + WorkflowRuntimeProxyStateListView +) +from .views.workflow_template_views import ( + DocumentTypeWorkflowTemplatesView, ToolLaunchWorkflows, + WorkflowTemplateCreateView, WorkflowTemplateDeleteView, + WorkflowTemplateEditView, WorkflowTemplateListView, + WorkflowTemplatePreviewView, WorkflowTemplateDocumentTypesView +) +from .views.workflow_template_state_views import ( + WorkflowTemplateStateActionCreateView, + WorkflowTemplateStateActionDeleteView, WorkflowTemplateStateActionEditView, + WorkflowTemplateStateActionListView, + WorkflowTemplateStateActionSelectionView, WorkflowTemplateStateCreateView, + WorkflowTemplateStateDeleteView, WorkflowTemplateStateEditView, + WorkflowTemplateStateListView +) +from .views.workflow_template_transition_views import ( + WorkflowTemplateTransitionCreateView, WorkflowTemplateTransitionDeleteView, + WorkflowTemplateTransitionEditView, WorkflowTemplateTransitionListView, + WorkflowTemplateTransitionTriggerEventListView, + WorkflowTemplateTransitionFieldCreateView, + WorkflowTemplateTransitionFieldDeleteView, + WorkflowTemplateTransitionFieldEditView, + WorkflowTemplateTransitionFieldListView ) -urlpatterns_workflows = [ +urlpatterns_workflow_instances = [ url( - regex=r'^document_type/(?P\d+)/workflows/$', - view=SetupDocumentTypeWorkflowsView.as_view(), - name='document_type_workflows' + regex=r'^documents/(?P\d+)/workflows/$', + view=WorkflowInstanceListView.as_view(), + name='workflow_instance_list' + ), + url( + regex=r'^documents/workflows/(?P\d+)/$', + view=WorkflowInstanceDetailView.as_view(), + name='workflow_instance_detail' + ), + url( + regex=r'^documents/workflows/(?P\d+)/transitions/select/$', + view=WorkflowInstanceTransitionSelectView.as_view(), + name='workflow_instance_transition_selection' + ), + url( + regex=r'^documents/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', + view=WorkflowInstanceTransitionExecuteView.as_view(), + name='workflow_instance_transition_execute' + ), +] + +urlpatterns_workflow_runtime_proxies = [ + url( + regex=r'workflow_runtime_proxies/$', + view=WorkflowRuntimeProxyListView.as_view(), + name='workflow_runtime_proxy_list' + ), + url( + regex=r'^workflow_runtime_proxies/(?P\d+)/documents/$', + view=WorkflowRuntimeProxyDocumentListView.as_view(), + name='workflow_runtime_proxy_document_list' + ), + url( + regex=r'^workflow_runtime_proxies/(?P\d+)/states/$', + view=WorkflowRuntimeProxyStateListView.as_view(), + name='workflow_runtime_proxy_state_list' + ), + url( + regex=r'^workflow_runtime_proxies/states/(?P\d+)/documents/$', + view=WorkflowRuntimeProxyStateDocumentListView.as_view(), + name='workflow_runtime_proxy_state_document_list' + ), +] + +urlpatterns_workflow_states = [ + url( + regex=r'^workflow_templates/(?P\d+)/states/$', + view=WorkflowTemplateStateListView.as_view(), + name='workflow_template_state_list' + ), + url( + regex=r'^workflow_templates/(?P\d+)/states/create/$', + view=WorkflowTemplateStateCreateView.as_view(), + name='workflow_template_state_create' + ), + url( + regex=r'^workflow_templates/states/(?P\d+)/delete/$', + view=WorkflowTemplateStateDeleteView.as_view(), + name='workflow_template_state_delete' + ), + url( + regex=r'^workflow_templates/states/(?P\d+)/edit/$', + view=WorkflowTemplateStateEditView.as_view(), + name='workflow_template_state_edit' + ), +] + +urlpatterns_workflow_state_actions = [ + url( + regex=r'^workflow_templates/states/(?P\d+)/actions/$', + view=WorkflowTemplateStateActionListView.as_view(), + name='workflow_template_state_action_list' + ), + url( + regex=r'^workflow_templates/states/(?P\d+)/actions/selection/$', + view=WorkflowTemplateStateActionSelectionView.as_view(), + name='workflow_template_state_action_selection' + ), + url( + regex=r'^workflow_templates/states/(?P\d+)/actions/(?P[a-zA-Z0-9_.]+)/create/$', + view=WorkflowTemplateStateActionCreateView.as_view(), + name='workflow_template_state_action_create' + ), + url( + regex=r'^workflow_templates/states/actions/(?P\d+)/delete/$', + view=WorkflowTemplateStateActionDeleteView.as_view(), + name='workflow_template_state_action_delete' + ), + url( + regex=r'^workflow_templates/states/actions/(?P\d+)/edit/$', + view=WorkflowTemplateStateActionEditView.as_view(), + name='workflow_template_state_action_edit' + ), +] + +urlpatterns_workflow_templates = [ + url( + regex=r'^workflow_templates/$', view=WorkflowTemplateListView.as_view(), + name='workflow_template_list' + ), + url( + regex=r'^workflow_templates/create/$', view=WorkflowTemplateCreateView.as_view(), + name='workflow_template_create' + ), + url( + regex=r'^workflow_templates/(?P\d+)/delete/$', + view=WorkflowTemplateDeleteView.as_view(), name='workflow_template_delete' + ), + url( + regex=r'^workflow_templates/(?P\d+)/document_types/$', + view=WorkflowTemplateDocumentTypesView.as_view(), + name='workflow_template_document_types' + ), + url( + regex=r'^workflow_templates/(?P\d+)/edit/$', + view=WorkflowTemplateEditView.as_view(), name='workflow_template_edit' + ), + url( + regex=r'^workflow_templates/(?P\d+)/preview/$', + view=WorkflowTemplatePreviewView.as_view(), + name='workflow_template_preview' + ), + url( + regex=r'^document_types/(?P\d+)/workflow_templates/$', + view=DocumentTypeWorkflowTemplatesView.as_view(), + name='document_type_workflow_templates' + ), +] +urlpatterns_workflow_transitions = [ + url( + regex=r'^workflow_templates/(?P\d+)/transitions/$', + view=WorkflowTemplateTransitionListView.as_view(), + name='workflow_template_transition_list' + ), + url( + regex=r'^workflow_templates/(?P\d+)/transitions/create/$', + view=WorkflowTemplateTransitionCreateView.as_view(), + name='workflow_template_transition_create' + ), + url( + regex=r'^workflow_templates/(?P\d+)/transitions/events/$', + view=WorkflowTemplateTransitionTriggerEventListView.as_view(), + name='workflow_template_transition_events' + ), + url( + regex=r'^workflow_templates/transitions/(?P\d+)/delete/$', + view=WorkflowTemplateTransitionDeleteView.as_view(), + name='workflow_template_transition_delete' + ), + url( + regex=r'^workflow_templates/transitions/(?P\d+)/edit/$', + view=WorkflowTemplateTransitionEditView.as_view(), + name='workflow_template_transition_edit' ), ] urlpatterns_workflow_transition_fields = [ url( - regex=r'^setup/workflows/transitions/(?P\d+)/fields/create/$', - view=SetupWorkflowTransitionFieldCreateView.as_view(), - name='setup_workflow_transition_field_create' + regex=r'^workflow_templates/transitions/(?P\d+)/fields/create/$', + view=WorkflowTemplateTransitionFieldCreateView.as_view(), + name='workflow_template_transition_field_create' ), url( - regex=r'^setup/workflows/transitions/(?P\d+)/fields/$', - view=SetupWorkflowTransitionFieldListView.as_view(), - name='setup_workflow_transition_field_list' + regex=r'^workflow_templates/transitions/(?P\d+)/fields/$', + view=WorkflowTemplateTransitionFieldListView.as_view(), + name='workflow_template_transition_field_list' ), url( - regex=r'^setup/workflows/transitions/fields/(?P\d+)/delete/$', - view=SetupWorkflowTransitionFieldDeleteView.as_view(), - name='setup_workflow_transition_field_delete' + regex=r'^workflow_templates/transitions/fields/(?P\d+)/delete/$', + view=WorkflowTemplateTransitionFieldDeleteView.as_view(), + name='workflow_template_transition_field_delete' ), url( - regex=r'^setup/workflows/transitions/fields/(?P\d+)/edit/$', - view=SetupWorkflowTransitionFieldEditView.as_view(), - name='setup_workflow_transition_field_edit' + regex=r'^workflow_templates/transitions/fields/(?P\d+)/edit/$', + view=WorkflowTemplateTransitionFieldEditView.as_view(), + name='workflow_template_transition_field_edit' ), ] urlpatterns = [ url( - regex=r'^document/(?P\d+)/workflows/$', - view=DocumentWorkflowInstanceListView.as_view(), - name='document_workflow_instance_list' - ), - url( - regex=r'^document/workflows/(?P\d+)/$', - view=WorkflowInstanceDetailView.as_view(), - name='workflow_instance_detail' - ), - url( - regex=r'^document/workflows/(?P\d+)/transitions/select/$', - view=WorkflowInstanceTransitionSelectView.as_view(), - name='workflow_instance_transition_selection' - ), - url( - regex=r'^document/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', - view=WorkflowInstanceTransitionExecuteView.as_view(), - name='workflow_instance_transition_execute' - ), - url( - regex=r'^setup/all/$', view=SetupWorkflowListView.as_view(), - name='setup_workflow_list' - ), - url( - regex=r'^setup/create/$', view=SetupWorkflowCreateView.as_view(), - name='setup_workflow_create' - ), - url( - regex=r'^setup/workflow/(?P\d+)/edit/$', - view=SetupWorkflowEditView.as_view(), name='setup_workflow_edit' - ), - url( - regex=r'^setup/workflow/(?P\d+)/delete/$', - view=SetupWorkflowDeleteView.as_view(), name='setup_workflow_delete' - ), - url( - regex=r'^setup/workflow/(?P\d+)/documents/$', - view=WorkflowDocumentListView.as_view(), - name='setup_workflow_document_list' - ), - url( - regex=r'^setup/workflow/(?P\d+)/document_types/$', - view=SetupWorkflowDocumentTypesView.as_view(), - name='setup_workflow_document_types' - ), - url( - regex=r'^setup/workflow/(?P\d+)/states/$', - view=SetupWorkflowStateListView.as_view(), - name='setup_workflow_state_list' - ), - url( - regex=r'^setup/workflow/(?P\d+)/states/create/$', - view=SetupWorkflowStateCreateView.as_view(), - name='setup_workflow_state_create' - ), - url( - regex=r'^setup/workflow/(?P\d+)/transitions/$', - view=SetupWorkflowTransitionListView.as_view(), - name='setup_workflow_transition_list' - ), - url( - regex=r'^setup/workflow/(?P\d+)/transitions/create/$', - view=SetupWorkflowTransitionCreateView.as_view(), - name='setup_workflow_transition_create' - ), - url( - regex=r'^setup/workflow/(?P\d+)/transitions/events/$', - view=SetupWorkflowTransitionTriggerEventListView.as_view(), - name='setup_workflow_transition_events' - ), - url( - regex=r'^setup/workflow/state/(?P\d+)/delete/$', - view=SetupWorkflowStateDeleteView.as_view(), - name='setup_workflow_state_delete' - ), - url( - regex=r'^setup/workflow/state/(?P\d+)/edit/$', - view=SetupWorkflowStateEditView.as_view(), - name='setup_workflow_state_edit' - ), - url( - regex=r'^setup/workflow/state/(?P\d+)/actions/$', - view=SetupWorkflowStateActionListView.as_view(), - name='setup_workflow_state_action_list' - ), - url( - regex=r'^setup/workflow/state/(?P\d+)/actions/selection/$', - view=SetupWorkflowStateActionSelectionView.as_view(), - name='setup_workflow_state_action_selection' - ), - url( - regex=r'^setup/workflow/state/(?P\d+)/actions/(?P[a-zA-Z0-9_.]+)/create/$', - view=SetupWorkflowStateActionCreateView.as_view(), - name='setup_workflow_state_action_create' - ), - url( - regex=r'^setup/workflow/state/actions/(?P\d+)/delete/$', - view=SetupWorkflowStateActionDeleteView.as_view(), - name='setup_workflow_state_action_delete' - ), - url( - regex=r'^setup/workflow/state/actions/(?P\d+)/edit/$', - view=SetupWorkflowStateActionEditView.as_view(), - name='setup_workflow_state_action_edit' - ), - url( - regex=r'^setup/workflow/transitions/(?P\d+)/delete/$', - view=SetupWorkflowTransitionDeleteView.as_view(), - name='setup_workflow_transition_delete' - ), - url( - regex=r'^setup/workflow/transitions/(?P\d+)/edit/$', - view=SetupWorkflowTransitionEditView.as_view(), - name='setup_workflow_transition_edit' - ), - url( - regex=r'^tools/workflow/all/launch/$', - view=ToolLaunchAllWorkflows.as_view(), - name='tool_launch_all_workflows' - ), - url( - regex=r'all/$', - view=WorkflowListView.as_view(), - name='workflow_list' - ), - url( - regex=r'^(?P\d+)/documents/$', - view=WorkflowDocumentListView.as_view(), - name='workflow_document_list' - ), - url( - regex=r'^(?P\d+)/states/$', - view=WorkflowStateListView.as_view(), - name='workflow_state_list' - ), - url( - regex=r'^(?P\d+)/preview/$', - view=WorkflowPreviewView.as_view(), - name='workflow_preview' - ), - url( - regex=r'^state/(?P\d+)/documents/$', - view=WorkflowStateDocumentListView.as_view(), - name='workflow_state_document_list' + regex=r'^tools/workflows/launch/$', + view=ToolLaunchWorkflows.as_view(), + name='tool_launch_workflows' ), ] -urlpatterns.extend(urlpatterns_workflows) +urlpatterns.extend(urlpatterns_workflow_instances) +urlpatterns.extend(urlpatterns_workflow_runtime_proxies) +urlpatterns.extend(urlpatterns_workflow_states) +urlpatterns.extend(urlpatterns_workflow_state_actions) +urlpatterns.extend(urlpatterns_workflow_templates) +urlpatterns.extend(urlpatterns_workflow_transitions) urlpatterns.extend(urlpatterns_workflow_transition_fields) api_urls = [ url( - regex=r'^workflows/$', view=APIWorkflowListView.as_view(), + regex=r'^workflows/$', view=APIWorkflowRuntimeProxyListView.as_view(), name='workflow-list' ), url( @@ -274,7 +299,7 @@ api_urls = [ ), url( regex=r'^document_types/(?P[0-9]+)/workflows/$', - view=APIDocumentTypeWorkflowListView.as_view(), + view=APIDocumentTypeWorkflowRuntimeProxyListView.as_view(), name='documenttype-workflow-list' ), ] diff --git a/mayan/apps/document_states/views/__init__.py b/mayan/apps/document_states/views/__init__.py index d6e36751b2..8b13789179 100644 --- a/mayan/apps/document_states/views/__init__.py +++ b/mayan/apps/document_states/views/__init__.py @@ -1,3 +1 @@ -from .workflow_instance_views import * # NOQA -from .workflow_proxy_views import * # NOQA -from .workflow_views import * # NOQA + diff --git a/mayan/apps/document_states/views/workflow_instance_views.py b/mayan/apps/document_states/views/workflow_instance_views.py index bed57fdcbc..6fcfd38fbd 100644 --- a/mayan/apps/document_states/views/workflow_instance_views.py +++ b/mayan/apps/document_states/views/workflow_instance_views.py @@ -14,20 +14,14 @@ from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.models import Document from ..forms import WorkflowInstanceTransitionSelectForm -from ..icons import icon_workflow_instance_detail, icon_workflow_list +from ..icons import icon_workflow_instance_detail, icon_workflow_template_list from ..links import link_workflow_instance_transition from ..literals import FIELD_TYPE_MAPPING, WIDGET_CLASS_MAPPING from ..models import WorkflowInstance from ..permissions import permission_workflow_view -__all__ = ( - 'DocumentWorkflowInstanceListView', 'WorkflowInstanceDetailView', - 'WorkflowInstanceTransitionSelectView', - 'WorkflowInstanceTransitionExecuteView' -) - -class DocumentWorkflowInstanceListView(SingleObjectListView): +class WorkflowInstanceListView(SingleObjectListView): def dispatch(self, request, *args, **kwargs): AccessControlList.objects.check_access( obj=self.get_document(), permissions=(permission_workflow_view,), @@ -35,7 +29,7 @@ class DocumentWorkflowInstanceListView(SingleObjectListView): ) return super( - DocumentWorkflowInstanceListView, self + WorkflowInstanceListView, self ).dispatch(request, *args, **kwargs) def get_document(self): @@ -44,7 +38,7 @@ class DocumentWorkflowInstanceListView(SingleObjectListView): def get_extra_context(self): return { 'hide_link': True, - 'no_results_icon': icon_workflow_list, + 'no_results_icon': icon_workflow_template_list, 'no_results_text': _( 'Assign workflows to the document type of this document ' 'to have this document execute those workflows. ' diff --git a/mayan/apps/document_states/views/workflow_proxy_views.py b/mayan/apps/document_states/views/workflow_proxy_views.py index 21efa2bb9e..77bd864ea4 100644 --- a/mayan/apps/document_states/views/workflow_proxy_views.py +++ b/mayan/apps/document_states/views/workflow_proxy_views.py @@ -9,18 +9,13 @@ from mayan.apps.common.generics import SingleObjectListView from mayan.apps.documents.models import Document from mayan.apps.documents.views import DocumentListView -from ..icons import icon_workflow_list -from ..links import link_setup_workflow_create, link_setup_workflow_state_create +from ..icons import icon_workflow_template_list +from ..links import link_workflow_template_create, link_workflow_template_state_create from ..models import WorkflowRuntimeProxy, WorkflowStateRuntimeProxy from ..permissions import permission_workflow_view -__all__ = ( - 'WorkflowDocumentListView', 'WorkflowListView', - 'WorkflowStateDocumentListView', 'WorkflowStateListView' -) - -class WorkflowDocumentListView(DocumentListView): +class WorkflowRuntimeProxyDocumentListView(DocumentListView): def dispatch(self, request, *args, **kwargs): self.workflow = get_object_or_404( klass=WorkflowRuntimeProxy, pk=self.kwargs['pk'] @@ -32,14 +27,14 @@ class WorkflowDocumentListView(DocumentListView): ) return super( - WorkflowDocumentListView, self + WorkflowRuntimeProxyDocumentListView, self ).dispatch(request, *args, **kwargs) def get_document_queryset(self): return Document.objects.filter(workflows__workflow=self.workflow) def get_extra_context(self): - context = super(WorkflowDocumentListView, self).get_extra_context() + context = super(WorkflowRuntimeProxyDocumentListView, self).get_extra_context() context.update( { 'no_results_text': _( @@ -56,14 +51,14 @@ class WorkflowDocumentListView(DocumentListView): return context -class WorkflowListView(SingleObjectListView): +class WorkflowRuntimeProxyListView(SingleObjectListView): object_permission = permission_workflow_view def get_extra_context(self): return { 'hide_object': True, - 'no_results_icon': icon_workflow_list, - 'no_results_main_link': link_setup_workflow_create.resolve( + 'no_results_icon': icon_workflow_template_list, + 'no_results_main_link': link_workflow_template_create.resolve( context=RequestContext(request=self.request) ), 'no_results_text': _( @@ -79,13 +74,13 @@ class WorkflowListView(SingleObjectListView): return WorkflowRuntimeProxy.objects.all() -class WorkflowStateDocumentListView(DocumentListView): +class WorkflowRuntimeProxyStateDocumentListView(DocumentListView): def get_document_queryset(self): return self.get_workflow_state().get_documents() def get_extra_context(self): workflow_state = self.get_workflow_state() - context = super(WorkflowStateDocumentListView, self).get_extra_context() + context = super(WorkflowRuntimeProxyStateDocumentListView, self).get_extra_context() context.update( { 'object': workflow_state, @@ -118,7 +113,7 @@ class WorkflowStateDocumentListView(DocumentListView): return workflow_state -class WorkflowStateListView(SingleObjectListView): +class WorkflowRuntimeProxyStateListView(SingleObjectListView): def dispatch(self, request, *args, **kwargs): AccessControlList.objects.check_access( obj=self.get_workflow(), permissions=(permission_workflow_view,), @@ -126,14 +121,14 @@ class WorkflowStateListView(SingleObjectListView): ) return super( - WorkflowStateListView, self + WorkflowRuntimeProxyStateListView, self ).dispatch(request, *args, **kwargs) def get_extra_context(self): return { 'hide_link': True, 'hide_object': True, - 'no_results_main_link': link_setup_workflow_state_create.resolve( + 'no_results_main_link': link_workflow_template_state_create.resolve( context=RequestContext( request=self.request, dict_={'object': self.get_workflow()} ) diff --git a/mayan/apps/document_states/views/workflow_template_state_views.py b/mayan/apps/document_states/views/workflow_template_state_views.py new file mode 100644 index 0000000000..a0f1e3cec7 --- /dev/null +++ b/mayan/apps/document_states/views/workflow_template_state_views.py @@ -0,0 +1,326 @@ +from __future__ import absolute_import, unicode_literals + +from django.contrib import messages +from django.db import transaction +from django.http import Http404, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.template import RequestContext +from django.urls import reverse, reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.generics import ( + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectDetailView, + SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + SingleObjectEditView, SingleObjectListView +) +from mayan.apps.common.mixins import ExternalObjectMixin +from mayan.apps.documents.events import event_document_type_edited +from mayan.apps.documents.models import DocumentType +from mayan.apps.documents.permissions import permission_document_type_edit +from mayan.apps.events.classes import EventType +from mayan.apps.events.models import StoredEventType + +from ..classes import WorkflowAction +from ..events import event_workflow_edited +from ..forms import ( + WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, + WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, + WorkflowTransitionTriggerEventRelationshipFormSet +) +from ..icons import ( + icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, + icon_workflow_transition, icon_workflow_transition_field +) +from ..links import ( + link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_action_selection, + link_workflow_template_transition_create, + link_workflow_template_transition_field_create, +) +from ..models import ( + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField +) +from ..permissions import ( + permission_workflow_create, permission_workflow_delete, + permission_workflow_edit, permission_workflow_tools, + permission_workflow_view, +) +from ..tasks import task_launch_all_workflows + + +class WorkflowTemplateStateActionCreateView(SingleObjectDynamicFormCreateView): + form_class = WorkflowStateActionDynamicForm + object_permission = permission_workflow_edit + + def get_class(self): + try: + return WorkflowAction.get(name=self.kwargs['class_path']) + except KeyError: + raise Http404( + '{} class not found'.format(self.kwargs['class_path']) + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow'), + 'object': self.get_object(), + 'title': _( + 'Create a "%s" workflow action' + ) % self.get_class().label, + 'workflow': self.get_object().workflow + } + + def get_form_extra_kwargs(self): + return { + 'request': self.request, + 'action_path': self.kwargs['class_path'] + } + + def get_form_schema(self): + return self.get_class()().get_form_schema(request=self.request) + + def get_instance_extra_data(self): + return { + 'action_path': self.kwargs['class_path'], + 'state': self.get_object() + } + + def get_object(self): + return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_state_action_list', + kwargs={'pk': self.get_object().pk} + ) + + +class WorkflowTemplateStateActionDeleteView(SingleObjectDeleteView): + model = WorkflowStateAction + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_state', 'workflow' + ), + 'object': self.get_object(), + 'title': _('Delete workflow state action: %s') % self.get_object(), + 'workflow': self.get_object().state.workflow, + 'workflow_state': self.get_object().state, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_state_action_list', + kwargs={'pk': self.get_object().state.pk} + ) + + +class WorkflowTemplateStateActionEditView(SingleObjectDynamicFormEditView): + form_class = WorkflowStateActionDynamicForm + model = WorkflowStateAction + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_state', 'workflow' + ), + 'object': self.get_object(), + 'title': _('Edit workflow state action: %s') % self.get_object(), + 'workflow': self.get_object().state.workflow, + 'workflow_state': self.get_object().state, + } + + def get_form_extra_kwargs(self): + return { + 'request': self.request, + 'action_path': self.get_object().action_path, + } + + def get_form_schema(self): + return self.get_object().get_class_instance().get_form_schema( + request=self.request + ) + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_state_action_list', + kwargs={'pk': self.get_object().state.pk} + ) + + +class WorkflowTemplateStateActionListView(SingleObjectListView): + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'hide_object': True, + 'navigation_object_list': ('object', 'workflow'), + 'no_results_icon': icon_workflow_state_action, + 'no_results_main_link': link_workflow_template_state_action_selection.resolve( + context=RequestContext( + request=self.request, dict_={ + 'object': self.get_workflow_state() + } + ) + ), + 'no_results_text': _( + 'Workflow state actions are macros that get executed when ' + 'documents enters or leaves the state in which they reside.' + ), + 'no_results_title': _( + 'There are no actions for this workflow state' + ), + 'object': self.get_workflow_state(), + 'title': _( + 'Actions for workflow state: %s' + ) % self.get_workflow_state(), + 'workflow': self.get_workflow_state().workflow, + } + + def get_form_schema(self): + return {'fields': self.get_class().fields} + + def get_source_queryset(self): + return self.get_workflow_state().actions.all() + + def get_workflow_state(self): + return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) + + +class WorkflowTemplateStateActionSelectionView(FormView): + form_class = WorkflowActionSelectionForm + view_permission = permission_workflow_edit + + def form_valid(self, form): + klass = form.cleaned_data['klass'] + return HttpResponseRedirect( + redirect_to=reverse( + viewname='document_states:workflow_template_state_action_create', + kwargs={'pk': self.get_object().pk, 'class_path': klass} + ) + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow' + ), + 'object': self.get_object(), + 'title': _('New workflow state action selection'), + 'workflow': self.get_object().workflow, + } + + def get_object(self): + return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) + + +class WorkflowTemplateStateCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = Workflow + external_object_permission = permission_workflow_edit + external_object_pk_url_kwarg = 'pk' + form_class = WorkflowStateForm + + def get_extra_context(self): + return { + 'object': self.get_workflow(), + 'title': _( + 'Create states for workflow: %s' + ) % self.get_workflow() + } + + def get_instance_extra_data(self): + return {'workflow': self.get_workflow()} + + def get_source_queryset(self): + return self.get_workflow().states.all() + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_state_list', + kwargs={'pk': self.kwargs['pk']} + ) + + def get_workflow(self): + return self.external_object + + +class WorkflowTemplateStateDeleteView(SingleObjectDeleteView): + model = WorkflowState + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.get_object(), + 'title': _( + 'Delete workflow state: %s?' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_state_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateStateEditView(SingleObjectEditView): + form_class = WorkflowStateForm + model = WorkflowState + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.get_object(), + 'title': _( + 'Edit workflow state: %s' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_state_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateStateListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = Workflow + external_object_permission = permission_workflow_view + external_object_pk_url_kwarg = 'pk' + object_permission = permission_workflow_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'no_results_icon': icon_workflow_state, + 'no_results_main_link': link_workflow_template_state_create.resolve( + context=RequestContext( + self.request, {'object': self.get_workflow()} + ) + ), + 'no_results_text': _( + 'Create states and link them using transitions.' + ), + 'no_results_title': _( + 'This workflow doesn\'t have any states' + ), + 'object': self.get_workflow(), + 'title': _('States of workflow: %s') % self.get_workflow() + } + + def get_source_queryset(self): + return self.get_workflow().states.all() + + def get_workflow(self): + return self.external_object diff --git a/mayan/apps/document_states/views/workflow_template_transition_views.py b/mayan/apps/document_states/views/workflow_template_transition_views.py new file mode 100644 index 0000000000..e56a10a625 --- /dev/null +++ b/mayan/apps/document_states/views/workflow_template_transition_views.py @@ -0,0 +1,372 @@ +from __future__ import absolute_import, unicode_literals + +from django.contrib import messages +from django.db import transaction +from django.http import Http404, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.template import RequestContext +from django.urls import reverse, reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.generics import ( + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectDetailView, + SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + SingleObjectEditView, SingleObjectListView +) +from mayan.apps.common.mixins import ExternalObjectMixin +from mayan.apps.documents.events import event_document_type_edited +from mayan.apps.documents.models import DocumentType +from mayan.apps.documents.permissions import permission_document_type_edit +from mayan.apps.events.classes import EventType +from mayan.apps.events.models import StoredEventType + +from ..classes import WorkflowAction +from ..events import event_workflow_edited +from ..forms import ( + WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, + WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, + WorkflowTransitionTriggerEventRelationshipFormSet +) +from ..icons import ( + icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, + icon_workflow_transition, icon_workflow_transition_field +) +from ..links import ( + link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_action_selection, + link_workflow_template_transition_create, + link_workflow_template_transition_field_create, +) +from ..models import ( + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField +) +from ..permissions import ( + permission_workflow_create, permission_workflow_delete, + permission_workflow_edit, permission_workflow_tools, + permission_workflow_view, +) +from ..tasks import task_launch_all_workflows + + +class WorkflowTemplateTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = Workflow + external_object_permission = permission_workflow_edit + external_object_pk_url_kwarg = 'pk' + form_class = WorkflowTransitionForm + + def get_extra_context(self): + return { + 'object': self.get_workflow(), + 'title': _( + 'Create transitions for workflow: %s' + ) % self.get_workflow() + } + + def get_form_kwargs(self): + kwargs = super( + WorkflowTemplateTransitionCreateView, self + ).get_form_kwargs() + kwargs['workflow'] = self.get_workflow() + return kwargs + + def get_instance_extra_data(self): + return {'workflow': self.get_workflow()} + + def get_source_queryset(self): + return self.get_workflow().transitions.all() + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.kwargs['pk']} + ) + + def get_workflow(self): + return self.external_object + + +class WorkflowTemplateTransitionDeleteView(SingleObjectDeleteView): + model = WorkflowTransition + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'object': self.get_object(), + 'navigation_object_list': ('object', 'workflow_instance'), + 'title': _( + 'Delete workflow transition: %s?' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateTransitionEditView(SingleObjectEditView): + form_class = WorkflowTransitionForm + model = WorkflowTransition + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.get_object(), + 'title': _( + 'Edit workflow transition: %s' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_form_kwargs(self): + kwargs = super( + WorkflowTemplateTransitionEditView, self + ).get_form_kwargs() + kwargs['workflow'] = self.get_object().workflow + return kwargs + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateTransitionListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = Workflow + external_object_permission = permission_workflow_view + external_object_pk_url_kwarg = 'pk' + object_permission = permission_workflow_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'no_results_icon': icon_workflow_transition, + 'no_results_main_link': link_workflow_template_transition_create.resolve( + context=RequestContext( + self.request, {'object': self.get_workflow()} + ) + ), + 'no_results_text': _( + 'Create a transition and use it to move a workflow from ' + ' one state to another.' + ), + 'no_results_title': _( + 'This workflow doesn\'t have any transitions' + ), + 'object': self.get_workflow(), + 'title': _( + 'Transitions of workflow: %s' + ) % self.get_workflow() + } + + def get_source_queryset(self): + return self.get_workflow().transitions.all() + + def get_workflow(self): + return self.external_object + + +class WorkflowTemplateTransitionTriggerEventListView(ExternalObjectMixin, FormView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + external_object_pk_url_kwarg = 'pk' + form_class = WorkflowTransitionTriggerEventRelationshipFormSet + + def dispatch(self, *args, **kwargs): + EventType.refresh() + return super( + WorkflowTemplateTransitionTriggerEventListView, self + ).dispatch(*args, **kwargs) + + def form_valid(self, form): + try: + for instance in form: + instance.save() + except Exception as exception: + messages.error( + message=_( + 'Error updating workflow transition trigger events; %s' + ) % exception, request=self.request + + ) + else: + messages.success( + message=_( + 'Workflow transition trigger events updated successfully' + ), request=self.request + ) + + return super( + WorkflowTemplateTransitionTriggerEventListView, self + ).form_valid(form=form) + + def get_extra_context(self): + return { + 'form_display_mode_table': True, + 'navigation_object_list': ('object', 'workflow'), + 'object': self.get_object(), + 'subtitle': _( + 'Triggers are events that cause this transition to execute ' + 'automatically.' + ), + 'title': _( + 'Workflow transition trigger events for: %s' + ) % self.get_object(), + 'workflow': self.get_object().workflow, + } + + def get_initial(self): + obj = self.get_object() + initial = [] + + # Return the queryset by name from the sorted list of the class + event_type_ids = [event_type.id for event_type in EventType.all()] + event_type_queryset = StoredEventType.objects.filter( + name__in=event_type_ids + ) + + # Sort queryset in Python by namespace, then by label + event_type_queryset = sorted( + event_type_queryset, key=lambda x: (x.namespace, x.label) + ) + + for event_type in event_type_queryset: + initial.append({ + 'transition': obj, + 'event_type': event_type, + }) + return initial + + def get_object(self): + return self.external_object + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateTransitionFieldCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + fields = ( + 'name', 'label', 'field_type', 'help_text', 'required', 'widget', + 'widget_kwargs' + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ('transition', 'workflow'), + 'transition': self.external_object, + 'title': _( + 'Create a field for workflow transition: %s' + ) % self.external_object, + 'workflow': self.external_object.workflow + } + + def get_instance_extra_data(self): + return { + 'transition': self.external_object, + } + + def get_queryset(self): + return self.external_object.fields.all() + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_field_list', + kwargs={'pk': self.external_object.pk} + ) + + +class WorkflowTemplateTransitionFieldDeleteView(SingleObjectDeleteView): + model = WorkflowTransitionField + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_transition', 'workflow' + ), + 'object': self.object, + 'title': _('Delete workflow transition field: %s') % self.object, + 'workflow': self.object.transition.workflow, + 'workflow_transition': self.object.transition, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_field_list', + kwargs={'pk': self.object.transition.pk} + ) + + +class WorkflowTemplateTransitionFieldEditView(SingleObjectEditView): + fields = ( + 'name', 'label', 'field_type', 'help_text', 'required', 'widget', + 'widget_kwargs' + ) + model = WorkflowTransitionField + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_transition', 'workflow' + ), + 'object': self.object, + 'title': _('Edit workflow transition field: %s') % self.object, + 'workflow': self.object.transition.workflow, + 'workflow_transition': self.object.transition, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_field_list', + kwargs={'pk': self.object.transition.pk} + ) + + +class WorkflowTemplateTransitionFieldListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'hide_object': True, + 'navigation_object_list': ('object', 'workflow'), + 'no_results_icon': icon_workflow_transition_field, + 'no_results_main_link': link_workflow_template_transition_field_create.resolve( + context=RequestContext( + request=self.request, dict_={ + 'object': self.external_object + } + ) + ), + 'no_results_text': _( + 'Workflow transition fields allow adding data to the ' + 'workflow\'s context. This additional context data can then ' + 'be used by other elements of the workflow system like the ' + 'workflow state actions.' + ), + 'no_results_title': _( + 'There are no fields for this workflow transition' + ), + 'object': self.external_object, + 'title': _( + 'Fields for workflow transition: %s' + ) % self.external_object, + 'workflow': self.external_object.workflow, + } + + def get_source_queryset(self): + return self.external_object.fields.all() diff --git a/mayan/apps/document_states/views/workflow_template_views.py b/mayan/apps/document_states/views/workflow_template_views.py new file mode 100644 index 0000000000..e755f42e33 --- /dev/null +++ b/mayan/apps/document_states/views/workflow_template_views.py @@ -0,0 +1,261 @@ +from __future__ import absolute_import, unicode_literals + +from django.contrib import messages +from django.db import transaction +from django.http import Http404, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.template import RequestContext +from django.urls import reverse, reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.generics import ( + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectDetailView, + SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + SingleObjectEditView, SingleObjectListView +) +from mayan.apps.common.mixins import ExternalObjectMixin +from mayan.apps.documents.events import event_document_type_edited +from mayan.apps.documents.models import DocumentType +from mayan.apps.documents.permissions import permission_document_type_edit +from mayan.apps.events.classes import EventType +from mayan.apps.events.models import StoredEventType + +from ..classes import WorkflowAction +from ..events import event_workflow_edited +from ..forms import ( + WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, + WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, + WorkflowTransitionTriggerEventRelationshipFormSet +) +from ..icons import ( + icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, + icon_workflow_transition, icon_workflow_transition_field +) +from ..links import ( + link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_action_selection, + link_workflow_template_transition_create, + link_workflow_template_transition_field_create, +) +from ..models import ( + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField +) +from ..permissions import ( + permission_workflow_create, permission_workflow_delete, + permission_workflow_edit, permission_workflow_tools, + permission_workflow_view, +) +from ..tasks import task_launch_all_workflows + + +class DocumentTypeWorkflowTemplatesView(AddRemoveView): + main_object_permission = permission_document_type_edit + main_object_model = DocumentType + main_object_pk_url_kwarg = 'pk' + secondary_object_model = Workflow + secondary_object_permission = permission_workflow_edit + list_available_title = _('Available workflows') + list_added_title = _('Workflows assigned this document type') + related_field = 'workflows' + + def get_actions_extra_kwargs(self): + return {'_user': self.request.user} + + def get_extra_context(self): + return { + 'object': self.main_object, + 'subtitle': _( + 'Removing a workflow from a document type will also ' + 'remove all running instances of that workflow.' + ), + 'title': _( + 'Workflows assigned the document type: %s' + ) % self.main_object, + } + + def action_add(self, queryset, _user): + with transaction.atomic(): + event_document_type_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.workflows.add(obj) + event_workflow_edited.commit( + action_object=self.main_object, actor=_user, target=obj + ) + + def action_remove(self, queryset, _user): + with transaction.atomic(): + event_document_type_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.workflows.remove(obj) + event_workflow_edited.commit( + action_object=self.main_object, actor=_user, + target=obj + ) + obj.instances.filter( + document__document_type=self.main_object + ).delete() + + +class WorkflowTemplateListView(SingleObjectListView): + model = Workflow + object_permission = permission_workflow_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'no_results_icon': icon_workflow_template_list, + 'no_results_main_link': link_workflow_template_create.resolve( + context=RequestContext(request=self.request) + ), + 'no_results_text': _( + 'Workflows store a series of states and keep track of the ' + 'current state of a document. Transitions are used to change the ' + 'current state to a new one.' + ), + 'no_results_title': _( + 'No workflows have been defined' + ), + 'title': _('Workflows'), + } + + +class WorkflowTemplateCreateView(SingleObjectCreateView): + extra_context = {'title': _('Create workflow')} + form_class = WorkflowForm + model = Workflow + post_action_redirect = reverse_lazy( + viewname='document_states:workflow_template_list' + ) + view_permission = permission_workflow_create + + def get_save_extra_data(self): + return {'_user': self.request.user} + + +class WorkflowTemplateDeleteView(SingleObjectDeleteView): + model = Workflow + object_permission = permission_workflow_delete + post_action_redirect = reverse_lazy( + viewname='document_states:workflow_template_list' + ) + + def get_extra_context(self): + return { + 'title': _( + 'Delete workflow: %s?' + ) % self.object, + } + + +class WorkflowTemplateEditView(SingleObjectEditView): + form_class = WorkflowForm + model = Workflow + object_permission = permission_workflow_edit + post_action_redirect = reverse_lazy( + viewname='document_states:workflow_template_list' + ) + + def get_extra_context(self): + return { + 'title': _( + 'Edit workflow: %s' + ) % self.object, + } + + def get_save_extra_data(self): + return {'_user': self.request.user} + + +class WorkflowTemplateDocumentTypesView(AddRemoveView): + main_object_permission = permission_workflow_edit + main_object_model = Workflow + main_object_pk_url_kwarg = 'pk' + secondary_object_model = DocumentType + secondary_object_permission = permission_document_type_edit + list_available_title = _('Available document types') + list_added_title = _('Document types assigned this workflow') + related_field = 'document_types' + + def get_actions_extra_kwargs(self): + return {'_user': self.request.user} + + def get_extra_context(self): + return { + 'object': self.main_object, + 'subtitle': _( + 'Removing a document type from a workflow will also ' + 'remove all running instances of that workflow for ' + 'documents of the document type just removed.' + ), + 'title': _( + 'Document types assigned the workflow: %s' + ) % self.main_object, + } + + def action_add(self, queryset, _user): + with transaction.atomic(): + event_workflow_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.document_types.add(obj) + event_document_type_edited.commit( + action_object=self.main_object, actor=_user, target=obj + ) + + def action_remove(self, queryset, _user): + with transaction.atomic(): + event_workflow_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.document_types.remove(obj) + event_document_type_edited.commit( + action_object=self.main_object, actor=_user, + target=obj + ) + self.main_object.instances.filter( + document__document_type=obj + ).delete() + + +class WorkflowTemplatePreviewView(SingleObjectDetailView): + form_class = WorkflowPreviewForm + model = Workflow + object_permission = permission_workflow_view + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _('Preview of: %s') % self.get_object() + } + + +class ToolLaunchWorkflows(ConfirmView): + extra_context = { + 'title': _('Launch all workflows?'), + 'subtitle': _( + 'This will launch all workflows created after documents have ' + 'already been uploaded.' + ) + } + view_permission = permission_workflow_tools + + def view_action(self): + task_launch_all_workflows.apply_async() + messages.success( + message=_('Workflow launch queued successfully.'), + request=self.request + ) diff --git a/mayan/apps/document_states/views/workflow_views.py b/mayan/apps/document_states/views/workflow_views.py deleted file mode 100644 index 008c83a108..0000000000 --- a/mayan/apps/document_states/views/workflow_views.py +++ /dev/null @@ -1,883 +0,0 @@ -from __future__ import absolute_import, unicode_literals - -from django.contrib import messages -from django.db import transaction -from django.http import Http404, HttpResponseRedirect -from django.shortcuts import get_object_or_404 -from django.template import RequestContext -from django.urls import reverse, reverse_lazy -from django.utils.translation import ugettext_lazy as _ - -from mayan.apps.common.generics import ( - AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, - SingleObjectDeleteView, SingleObjectDetailView, - SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, - SingleObjectEditView, SingleObjectListView -) -from mayan.apps.common.mixins import ExternalObjectMixin -from mayan.apps.documents.events import event_document_type_edited -from mayan.apps.documents.models import DocumentType -from mayan.apps.documents.permissions import permission_document_type_edit -from mayan.apps.events.classes import EventType -from mayan.apps.events.models import StoredEventType - -from ..classes import WorkflowAction -from ..events import event_workflow_edited -from ..forms import ( - WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, - WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, - WorkflowTransitionTriggerEventRelationshipFormSet -) -from ..icons import ( - icon_workflow_list, icon_workflow_state, icon_workflow_state_action, - icon_workflow_transition, icon_workflow_transition_field -) -from ..links import ( - link_setup_workflow_create, link_setup_workflow_state_create, - link_setup_workflow_state_action_selection, - link_setup_workflow_transition_create, - link_setup_workflow_transition_field_create, -) -from ..models import ( - Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, - WorkflowTransitionField -) -from ..permissions import ( - permission_workflow_create, permission_workflow_delete, - permission_workflow_edit, permission_workflow_tools, - permission_workflow_view, -) -from ..tasks import task_launch_all_workflows - -__all__ = ( - 'SetupWorkflowListView', 'SetupWorkflowCreateView', 'SetupWorkflowEditView', - 'SetupWorkflowDeleteView', 'SetupWorkflowDocumentTypesView', - 'SetupWorkflowStateActionCreateView', 'SetupWorkflowStateActionDeleteView', - 'SetupWorkflowStateActionEditView', 'SetupWorkflowStateActionListView', - 'SetupWorkflowStateActionSelectionView', 'SetupWorkflowStateCreateView', - 'SetupWorkflowStateDeleteView', 'SetupWorkflowStateEditView', - 'SetupWorkflowStateListView', 'SetupWorkflowTransitionCreateView', - 'SetupWorkflowTransitionDeleteView', 'SetupWorkflowTransitionEditView', - 'SetupWorkflowTransitionListView', - 'SetupWorkflowTransitionTriggerEventListView', 'ToolLaunchAllWorkflows', - 'WorkflowPreviewView' -) - - -class SetupDocumentTypeWorkflowsView(AddRemoveView): - main_object_permission = permission_document_type_edit - main_object_model = DocumentType - main_object_pk_url_kwarg = 'pk' - secondary_object_model = Workflow - secondary_object_permission = permission_workflow_edit - list_available_title = _('Available workflows') - list_added_title = _('Workflows assigned this document type') - related_field = 'workflows' - - def get_actions_extra_kwargs(self): - return {'_user': self.request.user} - - def get_extra_context(self): - return { - 'object': self.main_object, - 'subtitle': _( - 'Removing a workflow from a document type will also ' - 'remove all running instances of that workflow.' - ), - 'title': _( - 'Workflows assigned the document type: %s' - ) % self.main_object, - } - - def action_add(self, queryset, _user): - with transaction.atomic(): - event_document_type_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.workflows.add(obj) - event_workflow_edited.commit( - action_object=self.main_object, actor=_user, target=obj - ) - - def action_remove(self, queryset, _user): - with transaction.atomic(): - event_document_type_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.workflows.remove(obj) - event_workflow_edited.commit( - action_object=self.main_object, actor=_user, - target=obj - ) - obj.instances.filter( - document__document_type=self.main_object - ).delete() - - -class SetupWorkflowListView(SingleObjectListView): - model = Workflow - object_permission = permission_workflow_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'no_results_icon': icon_workflow_list, - 'no_results_main_link': link_setup_workflow_create.resolve( - context=RequestContext(request=self.request) - ), - 'no_results_text': _( - 'Workflows store a series of states and keep track of the ' - 'current state of a document. Transitions are used to change the ' - 'current state to a new one.' - ), - 'no_results_title': _( - 'No workflows have been defined' - ), - 'title': _('Workflows'), - } - - -class SetupWorkflowCreateView(SingleObjectCreateView): - extra_context = {'title': _('Create workflow')} - form_class = WorkflowForm - model = Workflow - post_action_redirect = reverse_lazy( - viewname='document_states:setup_workflow_list' - ) - view_permission = permission_workflow_create - - def get_save_extra_data(self): - return {'_user': self.request.user} - - -class SetupWorkflowDeleteView(SingleObjectDeleteView): - model = Workflow - object_permission = permission_workflow_delete - post_action_redirect = reverse_lazy( - viewname='document_states:setup_workflow_list' - ) - - def get_extra_context(self): - return { - 'title': _( - 'Delete workflow: %s?' - ) % self.object, - } - - -class SetupWorkflowEditView(SingleObjectEditView): - form_class = WorkflowForm - model = Workflow - object_permission = permission_workflow_edit - post_action_redirect = reverse_lazy( - viewname='document_states:setup_workflow_list' - ) - - def get_extra_context(self): - return { - 'title': _( - 'Edit workflow: %s' - ) % self.object, - } - - def get_save_extra_data(self): - return {'_user': self.request.user} - - -class SetupWorkflowDocumentTypesView(AddRemoveView): - main_object_permission = permission_workflow_edit - main_object_model = Workflow - main_object_pk_url_kwarg = 'pk' - secondary_object_model = DocumentType - secondary_object_permission = permission_document_type_edit - list_available_title = _('Available document types') - list_added_title = _('Document types assigned this workflow') - related_field = 'document_types' - - def get_actions_extra_kwargs(self): - return {'_user': self.request.user} - - def get_extra_context(self): - return { - 'object': self.main_object, - 'subtitle': _( - 'Removing a document type from a workflow will also ' - 'remove all running instances of that workflow for ' - 'documents of the document type just removed.' - ), - 'title': _( - 'Document types assigned the workflow: %s' - ) % self.main_object, - } - - def action_add(self, queryset, _user): - with transaction.atomic(): - event_workflow_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.document_types.add(obj) - event_document_type_edited.commit( - action_object=self.main_object, actor=_user, target=obj - ) - - def action_remove(self, queryset, _user): - with transaction.atomic(): - event_workflow_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.document_types.remove(obj) - event_document_type_edited.commit( - action_object=self.main_object, actor=_user, - target=obj - ) - self.main_object.instances.filter( - document__document_type=obj - ).delete() - - -# Workflow state actions - - -class SetupWorkflowStateActionCreateView(SingleObjectDynamicFormCreateView): - form_class = WorkflowStateActionDynamicForm - object_permission = permission_workflow_edit - - def get_class(self): - try: - return WorkflowAction.get(name=self.kwargs['class_path']) - except KeyError: - raise Http404( - '{} class not found'.format(self.kwargs['class_path']) - ) - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow'), - 'object': self.get_object(), - 'title': _( - 'Create a "%s" workflow action' - ) % self.get_class().label, - 'workflow': self.get_object().workflow - } - - def get_form_extra_kwargs(self): - return { - 'request': self.request, - 'action_path': self.kwargs['class_path'] - } - - def get_form_schema(self): - return self.get_class()().get_form_schema(request=self.request) - - def get_instance_extra_data(self): - return { - 'action_path': self.kwargs['class_path'], - 'state': self.get_object() - } - - def get_object(self): - return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_state_action_list', - kwargs={'pk': self.get_object().pk} - ) - - -class SetupWorkflowStateActionDeleteView(SingleObjectDeleteView): - model = WorkflowStateAction - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_state', 'workflow' - ), - 'object': self.get_object(), - 'title': _('Delete workflow state action: %s') % self.get_object(), - 'workflow': self.get_object().state.workflow, - 'workflow_state': self.get_object().state, - } - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_state_action_list', - kwargs={'pk': self.get_object().state.pk} - ) - - -class SetupWorkflowStateActionEditView(SingleObjectDynamicFormEditView): - form_class = WorkflowStateActionDynamicForm - model = WorkflowStateAction - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_state', 'workflow' - ), - 'object': self.get_object(), - 'title': _('Edit workflow state action: %s') % self.get_object(), - 'workflow': self.get_object().state.workflow, - 'workflow_state': self.get_object().state, - } - - def get_form_extra_kwargs(self): - return { - 'request': self.request, - 'action_path': self.get_object().action_path, - } - - def get_form_schema(self): - return self.get_object().get_class_instance().get_form_schema( - request=self.request - ) - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_state_action_list', - kwargs={'pk': self.get_object().state.pk} - ) - - -class SetupWorkflowStateActionListView(SingleObjectListView): - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'hide_object': True, - 'navigation_object_list': ('object', 'workflow'), - 'no_results_icon': icon_workflow_state_action, - 'no_results_main_link': link_setup_workflow_state_action_selection.resolve( - context=RequestContext( - request=self.request, dict_={ - 'object': self.get_workflow_state() - } - ) - ), - 'no_results_text': _( - 'Workflow state actions are macros that get executed when ' - 'documents enters or leaves the state in which they reside.' - ), - 'no_results_title': _( - 'There are no actions for this workflow state' - ), - 'object': self.get_workflow_state(), - 'title': _( - 'Actions for workflow state: %s' - ) % self.get_workflow_state(), - 'workflow': self.get_workflow_state().workflow, - } - - def get_form_schema(self): - return {'fields': self.get_class().fields} - - def get_source_queryset(self): - return self.get_workflow_state().actions.all() - - def get_workflow_state(self): - return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) - - -class SetupWorkflowStateActionSelectionView(FormView): - form_class = WorkflowActionSelectionForm - view_permission = permission_workflow_edit - - def form_valid(self, form): - klass = form.cleaned_data['klass'] - return HttpResponseRedirect( - redirect_to=reverse( - viewname='document_states:setup_workflow_state_action_create', - kwargs={'pk': self.get_object().pk, 'class_path': klass} - ) - ) - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow' - ), - 'object': self.get_object(), - 'title': _('New workflow state action selection'), - 'workflow': self.get_object().workflow, - } - - def get_object(self): - return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) - - -# Workflow states - - -class SetupWorkflowStateCreateView(ExternalObjectMixin, SingleObjectCreateView): - external_object_class = Workflow - external_object_permission = permission_workflow_edit - external_object_pk_url_kwarg = 'pk' - form_class = WorkflowStateForm - - def get_extra_context(self): - return { - 'object': self.get_workflow(), - 'title': _( - 'Create states for workflow: %s' - ) % self.get_workflow() - } - - def get_instance_extra_data(self): - return {'workflow': self.get_workflow()} - - def get_source_queryset(self): - return self.get_workflow().states.all() - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_state_list', - kwargs={'pk': self.kwargs['pk']} - ) - - def get_workflow(self): - return self.external_object - - -class SetupWorkflowStateDeleteView(SingleObjectDeleteView): - model = WorkflowState - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow_instance'), - 'object': self.get_object(), - 'title': _( - 'Delete workflow state: %s?' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_state_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowStateEditView(SingleObjectEditView): - form_class = WorkflowStateForm - model = WorkflowState - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow_instance'), - 'object': self.get_object(), - 'title': _( - 'Edit workflow state: %s' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_state_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowStateListView(ExternalObjectMixin, SingleObjectListView): - external_object_class = Workflow - external_object_permission = permission_workflow_view - external_object_pk_url_kwarg = 'pk' - object_permission = permission_workflow_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'no_results_icon': icon_workflow_state, - 'no_results_main_link': link_setup_workflow_state_create.resolve( - context=RequestContext( - self.request, {'object': self.get_workflow()} - ) - ), - 'no_results_text': _( - 'Create states and link them using transitions.' - ), - 'no_results_title': _( - 'This workflow doesn\'t have any states' - ), - 'object': self.get_workflow(), - 'title': _('States of workflow: %s') % self.get_workflow() - } - - def get_source_queryset(self): - return self.get_workflow().states.all() - - def get_workflow(self): - return self.external_object - - -# Transitions - - -class SetupWorkflowTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView): - external_object_class = Workflow - external_object_permission = permission_workflow_edit - external_object_pk_url_kwarg = 'pk' - form_class = WorkflowTransitionForm - - def get_extra_context(self): - return { - 'object': self.get_workflow(), - 'title': _( - 'Create transitions for workflow: %s' - ) % self.get_workflow() - } - - def get_form_kwargs(self): - kwargs = super( - SetupWorkflowTransitionCreateView, self - ).get_form_kwargs() - kwargs['workflow'] = self.get_workflow() - return kwargs - - def get_instance_extra_data(self): - return {'workflow': self.get_workflow()} - - def get_source_queryset(self): - return self.get_workflow().transitions.all() - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.kwargs['pk']} - ) - - def get_workflow(self): - return self.external_object - - -class SetupWorkflowTransitionDeleteView(SingleObjectDeleteView): - model = WorkflowTransition - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'object': self.get_object(), - 'navigation_object_list': ('object', 'workflow_instance'), - 'title': _( - 'Delete workflow transition: %s?' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowTransitionEditView(SingleObjectEditView): - form_class = WorkflowTransitionForm - model = WorkflowTransition - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow_instance'), - 'object': self.get_object(), - 'title': _( - 'Edit workflow transition: %s' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_form_kwargs(self): - kwargs = super( - SetupWorkflowTransitionEditView, self - ).get_form_kwargs() - kwargs['workflow'] = self.get_object().workflow - return kwargs - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowTransitionListView(ExternalObjectMixin, SingleObjectListView): - external_object_class = Workflow - external_object_permission = permission_workflow_view - external_object_pk_url_kwarg = 'pk' - object_permission = permission_workflow_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'no_results_icon': icon_workflow_transition, - 'no_results_main_link': link_setup_workflow_transition_create.resolve( - context=RequestContext( - self.request, {'object': self.get_workflow()} - ) - ), - 'no_results_text': _( - 'Create a transition and use it to move a workflow from ' - ' one state to another.' - ), - 'no_results_title': _( - 'This workflow doesn\'t have any transitions' - ), - 'object': self.get_workflow(), - 'title': _( - 'Transitions of workflow: %s' - ) % self.get_workflow() - } - - def get_source_queryset(self): - return self.get_workflow().transitions.all() - - def get_workflow(self): - return self.external_object - - -class SetupWorkflowTransitionTriggerEventListView(ExternalObjectMixin, FormView): - external_object_class = WorkflowTransition - external_object_permission = permission_workflow_edit - external_object_pk_url_kwarg = 'pk' - form_class = WorkflowTransitionTriggerEventRelationshipFormSet - - def dispatch(self, *args, **kwargs): - EventType.refresh() - return super( - SetupWorkflowTransitionTriggerEventListView, self - ).dispatch(*args, **kwargs) - - def form_valid(self, form): - try: - for instance in form: - instance.save() - except Exception as exception: - messages.error( - message=_( - 'Error updating workflow transition trigger events; %s' - ) % exception, request=self.request - - ) - else: - messages.success( - message=_( - 'Workflow transition trigger events updated successfully' - ), request=self.request - ) - - return super( - SetupWorkflowTransitionTriggerEventListView, self - ).form_valid(form=form) - - def get_extra_context(self): - return { - 'form_display_mode_table': True, - 'navigation_object_list': ('object', 'workflow'), - 'object': self.get_object(), - 'subtitle': _( - 'Triggers are events that cause this transition to execute ' - 'automatically.' - ), - 'title': _( - 'Workflow transition trigger events for: %s' - ) % self.get_object(), - 'workflow': self.get_object().workflow, - } - - def get_initial(self): - obj = self.get_object() - initial = [] - - # Return the queryset by name from the sorted list of the class - event_type_ids = [event_type.id for event_type in EventType.all()] - event_type_queryset = StoredEventType.objects.filter( - name__in=event_type_ids - ) - - # Sort queryset in Python by namespace, then by label - event_type_queryset = sorted( - event_type_queryset, key=lambda x: (x.namespace, x.label) - ) - - for event_type in event_type_queryset: - initial.append({ - 'transition': obj, - 'event_type': event_type, - }) - return initial - - def get_object(self): - return self.external_object - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -# Transition fields - -class SetupWorkflowTransitionFieldCreateView(ExternalObjectMixin, SingleObjectCreateView): - external_object_class = WorkflowTransition - external_object_permission = permission_workflow_edit - fields = ( - 'name', 'label', 'field_type', 'help_text', 'required', 'widget', - 'widget_kwargs' - ) - def get_extra_context(self): - return { - 'navigation_object_list': ('transition', 'workflow'), - 'transition': self.external_object, - 'title': _( - 'Create a field for workflow transition: %s' - ) % self.external_object, - 'workflow': self.external_object.workflow - } - - def get_instance_extra_data(self): - return { - 'transition': self.external_object, - } - - def get_queryset(self): - return self.external_object.fields.all() - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_field_list', - kwargs={'pk': self.external_object.pk} - ) - - -class SetupWorkflowTransitionFieldDeleteView(SingleObjectDeleteView): - model = WorkflowTransitionField - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_transition', 'workflow' - ), - 'object': self.object, - 'title': _('Delete workflow transition field: %s') % self.object, - 'workflow': self.object.transition.workflow, - 'workflow_transition': self.object.transition, - } - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_field_list', - kwargs={'pk': self.object.transition.pk} - ) - - -class SetupWorkflowTransitionFieldEditView(SingleObjectEditView): - fields = ( - 'name', 'label', 'field_type', 'help_text', 'required', 'widget', - 'widget_kwargs' - ) - model = WorkflowTransitionField - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_transition', 'workflow' - ), - 'object': self.object, - 'title': _('Edit workflow transition field: %s') % self.object, - 'workflow': self.object.transition.workflow, - 'workflow_transition': self.object.transition, - } - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_field_list', - kwargs={'pk': self.object.transition.pk} - ) - - -class SetupWorkflowTransitionFieldListView(ExternalObjectMixin, SingleObjectListView): - external_object_class = WorkflowTransition - external_object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'hide_object': True, - 'navigation_object_list': ('object', 'workflow'), - 'no_results_icon': icon_workflow_transition_field, - 'no_results_main_link': link_setup_workflow_transition_field_create.resolve( - context=RequestContext( - request=self.request, dict_={ - 'object': self.external_object - } - ) - ), - 'no_results_text': _( - 'Workflow transition fields allow adding data to the ' - 'workflow\'s context. This additional context data can then ' - 'be used by other elements of the workflow system like the ' - 'workflow state actions.' - ), - 'no_results_title': _( - 'There are no fields for this workflow transition' - ), - 'object': self.external_object, - 'title': _( - 'Fields for workflow transition: %s' - ) % self.external_object, - 'workflow': self.external_object.workflow, - } - - def get_source_queryset(self): - return self.external_object.fields.all() - - -class ToolLaunchAllWorkflows(ConfirmView): - extra_context = { - 'title': _('Launch all workflows?'), - 'subtitle': _( - 'This will launch all workflows created after documents have ' - 'already been uploaded.' - ) - } - view_permission = permission_workflow_tools - - def view_action(self): - task_launch_all_workflows.apply_async() - messages.success( - message=_('Workflow launch queued successfully.'), - request=self.request - ) - - -class WorkflowPreviewView(SingleObjectDetailView): - form_class = WorkflowPreviewForm - model = Workflow - object_permission = permission_workflow_view - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'hide_labels': True, - 'object': self.get_object(), - 'title': _('Preview of: %s') % self.get_object() - } diff --git a/mayan/apps/documents/migrations/0048_auto_20190711_0544.py b/mayan/apps/documents/migrations/0048_auto_20190711_0544.py new file mode 100644 index 0000000000..627329af90 --- /dev/null +++ b/mayan/apps/documents/migrations/0048_auto_20190711_0544.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-07-11 05:44 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.documents.models.document_version_models +import mayan.apps.storage.classes + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0047_auto_20180917_0737'), + ] + + operations = [ + migrations.AlterField( + model_name='documentversion', + name='file', + field=models.FileField(storage=mayan.apps.storage.classes.FakeStorageSubclass(), upload_to=mayan.apps.documents.models.document_version_models.UUID_FUNCTION, verbose_name='File'), + ), + ] diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index b5a1661bd8..afbf34cef3 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -229,7 +229,7 @@ class DocumentPage(models.Model): for transformation in transformations: converter.transform(transformation=transformation) - return page_image + return converter.get_page() except Exception as exception: # Cleanup in case of error logger.error( diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 19c0162ca8..a4d5014e91 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES @@ -18,15 +18,14 @@ setting_documentimagecache_storage = namespace.add_setting( default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'document image files.' - ), quoted=True + ) ) setting_documentimagecache_storage_arguments = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_cache') - ), help_text=_( + default={'location': os.path.join(settings.MEDIA_ROOT, 'document_cache')}, + help_text=_( 'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ), ) setting_disable_base_image_cache = namespace.add_setting( global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False, @@ -127,9 +126,8 @@ setting_storage_backend = namespace.add_setting( ) setting_storage_backend_arguments = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_storage') - ), help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.') + default={'location': os.path.join(settings.MEDIA_ROOT, 'document_storage')}, + help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.') ) setting_thumbnail_height = namespace.add_setting( global_name='DOCUMENTS_THUMBNAIL_HEIGHT', default='', help_text=_( diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index 95405e9242..a4e9132fde 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,13 +1,6 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_documentimagecache_storage, @@ -15,20 +8,10 @@ from .settings import ( setting_storage_backend, setting_storage_backend_arguments ) -storage_documentversion = import_string( +storage_documentversion = get_storage_subclass( dotted_path=setting_storage_backend.value -)( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_storage_backend_arguments.value) -storage_documentimagecache = import_string( +storage_documentimagecache = get_storage_subclass( dotted_path=setting_documentimagecache_storage.value -)( - **yaml.load( - stream=setting_documentimagecache_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_documentimagecache_storage_arguments.value) diff --git a/mayan/apps/events/apps.py b/mayan/apps/events/apps.py index 6ec59d3b90..046f98968b 100644 --- a/mayan/apps/events/apps.py +++ b/mayan/apps/events/apps.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.apps import MayanAppConfig from mayan.apps.common.html_widgets import TwoStateWidget from mayan.apps.common.menus import ( - menu_main, menu_object, menu_secondary, menu_tools, menu_topbar, menu_user + menu_object, menu_secondary, menu_tools, menu_topbar, menu_user ) from mayan.apps.navigation.classes import SourceColumn diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 79a0993916..cff210335e 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,12 +4,6 @@ import json import logging import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ @@ -57,11 +51,7 @@ class EXIFToolDriver(FileMetadataDriver): ) def read_settings(self): - driver_arguments = yaml.load( - stream=setting_drivers_arguments.value, Loader=SafeLoader - ) - - self.exiftool_path = driver_arguments.get( + self.exiftool_path = setting_drivers_arguments.value.get( 'exif_driver', {} ).get('exiftool_path', DEFAULT_EXIF_PATH) diff --git a/mayan/apps/file_metadata/settings.py b/mayan/apps/file_metadata/settings.py index 247f0b72ec..b0f8c19064 100644 --- a/mayan/apps/file_metadata/settings.py +++ b/mayan/apps/file_metadata/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EXIF_PATH @@ -16,12 +16,7 @@ setting_auto_process = namespace.add_setting( ) ) setting_drivers_arguments = namespace.add_setting( - default=''' - {{ - exif_driver: {{exiftool_path: {}}}, - - }} - '''.replace('\n', '').format(DEFAULT_EXIF_PATH), help_text=_( + default={'exif_driver': {'exiftool_path': DEFAULT_EXIF_PATH}}, help_text=_( 'Arguments to pass to the drivers.' - ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS', quoted=True + ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS' ) diff --git a/mayan/apps/lock_manager/settings.py b/mayan/apps/lock_manager/settings.py index d401b4c462..30448fa1fb 100644 --- a/mayan/apps/lock_manager/settings.py +++ b/mayan/apps/lock_manager/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_BACKEND, DEFAULT_LOCK_TIMEOUT_VALUE diff --git a/mayan/apps/mailer/literals.py b/mayan/apps/mailer/literals.py index b22405cee3..c76b51f7d3 100644 --- a/mayan/apps/mailer/literals.py +++ b/mayan/apps/mailer/literals.py @@ -8,11 +8,11 @@ DEFAULT_DOCUMENT_BODY_TEMPLATE = _( '--------\n ' 'This email has been sent from %(project_title)s (%(project_website)s)' ) - +DEFAULT_DOCUMENT_SUBJECT_TEMPLATE = _('Document: {{ document }}') DEFAULT_LINK_BODY_TEMPLATE = _( 'To access this document click on the following link: ' '{{ link }}\n\n--------\n ' 'This email has been sent from %(project_title)s (%(project_website)s)' ) - +DEFAULT_LINK_SUBJECT_TEMPLATE = _('Link for document: {{ document }}') EMAIL_SEPARATORS = (',', ';') diff --git a/mayan/apps/mailer/settings.py b/mayan/apps/mailer/settings.py index ed689b60a3..7fbf7f6b8b 100644 --- a/mayan/apps/mailer/settings.py +++ b/mayan/apps/mailer/settings.py @@ -2,31 +2,32 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( - DEFAULT_DOCUMENT_BODY_TEMPLATE, DEFAULT_LINK_BODY_TEMPLATE + DEFAULT_DOCUMENT_BODY_TEMPLATE, DEFAULT_DOCUMENT_SUBJECT_TEMPLATE, + DEFAULT_LINK_BODY_TEMPLATE, DEFAULT_LINK_SUBJECT_TEMPLATE ) namespace = Namespace(label=_('Mailing'), name='mailer') -setting_link_subject_template = namespace.add_setting( - default=_('Link for document: {{ document }}'), - help_text=_('Template for the document link email form subject line.'), - global_name='MAILER_LINK_SUBJECT_TEMPLATE', quoted=True -) -setting_link_body_template = namespace.add_setting( - default=DEFAULT_LINK_BODY_TEMPLATE, - help_text=_('Template for the document link email form body text. Can include HTML.'), - global_name='MAILER_LINK_BODY_TEMPLATE', quoted=True -) setting_document_subject_template = namespace.add_setting( - default=_('Document: {{ document }}'), + default=DEFAULT_DOCUMENT_SUBJECT_TEMPLATE, help_text=_('Template for the document email form subject line.'), - global_name='MAILER_DOCUMENT_SUBJECT_TEMPLATE', quoted=True + global_name='MAILER_DOCUMENT_SUBJECT_TEMPLATE' ) setting_document_body_template = namespace.add_setting( default=DEFAULT_DOCUMENT_BODY_TEMPLATE, help_text=_('Template for the document email form body text. Can include HTML.'), - global_name='MAILER_DOCUMENT_BODY_TEMPLATE', quoted=True + global_name='MAILER_DOCUMENT_BODY_TEMPLATE' +) +setting_link_subject_template = namespace.add_setting( + default=DEFAULT_LINK_SUBJECT_TEMPLATE, + help_text=_('Template for the document link email form subject line.'), + global_name='MAILER_LINK_SUBJECT_TEMPLATE' +) +setting_link_body_template = namespace.add_setting( + default=DEFAULT_LINK_BODY_TEMPLATE, + help_text=_('Template for the document link email form body text. Can include HTML.'), + global_name='MAILER_LINK_BODY_TEMPLATE' ) diff --git a/mayan/apps/mailer/tests/test_actions.py b/mayan/apps/mailer/tests/test_actions.py index eed6d9f49a..cf21f68b0d 100644 --- a/mayan/apps/mailer/tests/test_actions.py +++ b/mayan/apps/mailer/tests/test_actions.py @@ -139,7 +139,7 @@ class EmailActionViewTestCase(DocumentTestMixin, MailerTestMixin, WorkflowTestMi self._create_test_user_mailer() response = self.get( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.mailer.workflow_actions.EmailAction', @@ -151,7 +151,7 @@ class EmailActionViewTestCase(DocumentTestMixin, MailerTestMixin, WorkflowTestMi def _request_email_action_create_post_view(self): return self.post( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.mailer.workflow_actions.EmailAction', diff --git a/mayan/apps/metadata/settings.py b/mayan/apps/metadata/settings.py index ce7aa6de35..0387f45309 100644 --- a/mayan/apps/metadata/settings.py +++ b/mayan/apps/metadata/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .parsers import MetadataParser from .validators import MetadataValidator diff --git a/mayan/apps/mirroring/settings.py b/mayan/apps/mirroring/settings.py index aa16aa1335..41f9b372a5 100644 --- a/mayan/apps/mirroring/settings.py +++ b/mayan/apps/mirroring/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Mirroring'), name='mirroring') diff --git a/mayan/apps/navigation/classes.py b/mayan/apps/navigation/classes.py index beda911fb8..582a80fe82 100644 --- a/mayan/apps/navigation/classes.py +++ b/mayan/apps/navigation/classes.py @@ -369,10 +369,25 @@ class Menu(object): for resolved_navigation_object in resolved_navigation_object_list: resolved_links = [] + # List of resolved links source links used for deduplication + resolved_links_links = [] + for bound_source, links in self.bound_links.items(): try: if inspect.isclass(bound_source): if type(resolved_navigation_object) == bound_source: + # Check to see if object is a proxy model. If it is, add its parent model + # menu links too. + if hasattr(resolved_navigation_object, '_meta'): + parent_model = resolved_navigation_object._meta.proxy_for_model + if parent_model: + parent_instance = parent_model.objects.filter(pk=resolved_navigation_object.pk) + if parent_instance: + for link_set in self.resolve(context=context, source=parent_instance.first()): + for link in link_set['links']: + if link.link not in self.unbound_links.get(bound_source, ()): + resolved_links.append(link) + for link in links: resolved_link = link.resolve( context=context, @@ -395,10 +410,22 @@ class Menu(object): resolved_links.append(resolved_link) # No need for further content object match testing break + except TypeError: # When source is a dictionary pass + # Remove duplicated resolved link by using their source link + # instance as reference. The actual resolved link can't be used + # since a single source link can produce multiple resolved links. + # Since dictionaries keys can't have duplicates, we use that as a + # native deduplicator. + resolved_links_dict = {} + for resolved_link in resolved_links: + resolved_links_dict[resolved_link.link] = resolved_link + + resolved_links = resolved_links_dict.values() + if resolved_links: result.append( { @@ -407,6 +434,7 @@ class Menu(object): } ) + resolved_links = [] # View links for link in self.bound_links.get(current_view_name, []): diff --git a/mayan/apps/ocr/backends/tesseract.py b/mayan/apps/ocr/backends/tesseract.py index 3444198e4d..0a8f87cf2e 100644 --- a/mayan/apps/ocr/backends/tesseract.py +++ b/mayan/apps/ocr/backends/tesseract.py @@ -4,11 +4,6 @@ import logging import shutil import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -115,15 +110,10 @@ class Tesseract(OCRBackendBase): logger.debug('Available languages: %s', ', '.join(self.languages)) def read_settings(self): - backend_arguments = yaml.load( - Loader=SafeLoader, - stream=setting_ocr_backend_arguments.value or '{}', - ) - - self.tesseract_binary_path = backend_arguments.get( + self.tesseract_binary_path = setting_ocr_backend_arguments.value.get( 'tesseract_path', DEFAULT_TESSERACT_BINARY_PATH ) - self.command_timeout = backend_arguments.get( + self.command_timeout = setting_ocr_backend_arguments.value.get( 'timeout', DEFAULT_TESSERACT_TIMEOUT ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 1d8643819b..55e6a36a35 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,20 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( dotted_path=setting_ocr_backend.value -)( - **yaml.load( - stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader - ) -) +)(**setting_ocr_backend_arguments.value) diff --git a/mayan/apps/ocr/settings.py b/mayan/apps/ocr/settings.py index f2aa1052a3..51b92149aa 100644 --- a/mayan/apps/ocr/settings.py +++ b/mayan/apps/ocr/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('OCR'), name='ocr') @@ -13,7 +13,7 @@ setting_ocr_backend = namespace.add_setting( ) setting_ocr_backend_arguments = namespace.add_setting( global_name='OCR_BACKEND_ARGUMENTS', - default='' + default={} ) setting_auto_ocr = namespace.add_setting( global_name='OCR_AUTO_OCR', default=True, diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 1d0152442f..f716029806 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -2,16 +2,12 @@ from __future__ import absolute_import, unicode_literals import os -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.template import loader +from django.utils.html import mark_safe from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_dump, yaml_load from mayan.apps.common.settings import ( setting_celery_broker_url, setting_celery_result_backend ) @@ -24,9 +20,25 @@ class Variable(object): self.default = default self.environment_name = environment_name - def get_value(self): + def _get_value(self): return os.environ.get(self.environment_name, self.default) + def get_value(self): + return mark_safe(self._get_value()) + + +class YAMLVariable(Variable): + def _get_value(self): + value = os.environ.get(self.environment_name) + if value: + value = yaml_load(stream=value) + else: + value = self.default + + return yaml_dump( + data=value, allow_unicode=True, default_flow_style=True, width=999 + ).replace('...\n', '').replace('\n', '') + @python_2_unicode_compatible class PlatformTemplate(object): @@ -95,9 +107,7 @@ class PlatformTemplate(object): if context_string: context.update( - yaml.load( - stream=context_string, Loader=SafeLoader - ) + yaml_load(stream=context_string) ) return loader.render_to_string( template_name=self.get_template_name(), @@ -106,10 +116,6 @@ class PlatformTemplate(object): class PlatformTemplateSupervisord(PlatformTemplate): - context_defaults = { - 'BROKER_URL': 'redis://127.0.0.1:6379/0', - 'CELERY_RESULT_BACKEND': 'redis://127.0.0.1:6379/0', - } label = _('Template for Supervisord.') name = 'supervisord' settings = ( @@ -124,35 +130,38 @@ class PlatformTemplateSupervisord(PlatformTemplate): name='GUNICORN_TIMEOUT', default=120, environment_name='MAYAN_GUNICORN_TIMEOUT' ), - Variable( - name='DATABASE_CONN_MAX_AGE', default=0, - environment_name='MAYAN_DATABASE_CONN_MAX_AGE' - ), - Variable( - name='DATABASE_ENGINE', default='django.db.backends.postgresql', - environment_name='MAYAN_DATABASE_ENGINE' - ), - Variable( - name='DATABASE_HOST', default='127.0.0.1', - environment_name='MAYAN_DATABASE_HOST' - ), - Variable( - name='DATABASE_NAME', default='mayan', - environment_name='MAYAN_DATABASE_NAME' - ), - Variable( - name='DATABASE_PASSWORD', default='mayanuserpass', - environment_name='MAYAN_DATABASE_PASSWORD' - ), - Variable( - name='DATABASE_USER', default='mayan', - environment_name='MAYAN_DATABASE_USER' - ), Variable( name='INSTALLATION_PATH', default='/opt/mayan-edms', environment_name='MAYAN_INSTALLATION_PATH' ), - Variable( + YAMLVariable( + name='ALLOWED_HOSTS', + default=['*'], + environment_name='MAYAN_ALLOWED_HOSTS' + ), + YAMLVariable( + name='BROKER_URL', + default='redis://127.0.0.1:6379/0', + environment_name='MAYAN_BROKER_URL' + ), + YAMLVariable( + name='CELERY_RESULT_BACKEND', + default='redis://127.0.0.1:6379/0', + environment_name='MAYAN_CELERY_RESULT_BACKEND' + ), + YAMLVariable( + name='DATABASES', + default={ + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'mayan', 'PASSWORD':'mayanuserpass', + 'USER': 'mayan', 'HOST':'127.0.0.1' + } + }, + environment_name='MAYAN_DATABASES' + ), + YAMLVariable + ( name='MEDIA_ROOT', default='/opt/mayan-edms/media', environment_name='MAYAN_MEDIA_ROOT' ), diff --git a/mayan/apps/platform/templates/platform/supervisord.tmpl b/mayan/apps/platform/templates/platform/supervisord.tmpl index b87686c827..f6623d12de 100644 --- a/mayan/apps/platform/templates/platform/supervisord.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord.tmpl @@ -1,17 +1,12 @@ [supervisord] environment= - MAYAN_ALLOWED_HOSTS='["*"]', # Allow access to other network hosts other than localhost + PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, + DJANGO_SETTINGS_MODULE=mayan.settings.production, + MAYAN_MEDIA_ROOT="{{ MEDIA_ROOT }}", + MAYAN_ALLOWED_HOSTS="{{ ALLOWED_HOSTS }}", MAYAN_CELERY_RESULT_BACKEND="{{ CELERY_RESULT_BACKEND }}", MAYAN_BROKER_URL="{{ BROKER_URL }}", - PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, - MAYAN_MEDIA_ROOT={{ MEDIA_ROOT }}, - MAYAN_DATABASE_ENGINE={{ DATABASE_ENGINE }}, - MAYAN_DATABASE_HOST={{ DATABASE_HOST }}, - MAYAN_DATABASE_NAME={{ DATABASE_NAME }}, - MAYAN_DATABASE_PASSWORD={{ DATABASE_PASSWORD }}, - MAYAN_DATABASE_USER={{ DATABASE_USER }}, - MAYAN_DATABASE_CONN_MAX_AGE={{ DATABASE_CONN_MAX_AGE }}, - DJANGO_SETTINGS_MODULE=mayan.settings.production + MAYAN_DATABASES="{{ DATABASES }}" [program:mayan-gunicorn] autorestart = true diff --git a/mayan/apps/smart_settings/__init__.py b/mayan/apps/smart_settings/__init__.py index 143550fa7b..54f25908f0 100644 --- a/mayan/apps/smart_settings/__init__.py +++ b/mayan/apps/smart_settings/__init__.py @@ -1,5 +1,3 @@ from __future__ import unicode_literals -from .classes import Namespace, Setting # NOQA - default_app_config = 'mayan.apps.smart_settings.apps.SmartSettingsApp' diff --git a/mayan/apps/smart_settings/apps.py b/mayan/apps/smart_settings/apps.py index 32d2a69950..d84e9dedf8 100644 --- a/mayan/apps/smart_settings/apps.py +++ b/mayan/apps/smart_settings/apps.py @@ -11,6 +11,7 @@ from .links import ( link_namespace_detail, link_namespace_list, link_namespace_root_list, link_setting_edit ) +from .settings import * # NOQA from .widgets import setting_widget diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608..be064d0fa8 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -9,11 +9,6 @@ import sys import yaml -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.apps import apps from django.conf import settings from django.utils.functional import Promise @@ -21,6 +16,10 @@ from django.utils.encoding import ( force_bytes, force_text, python_2_unicode_compatible ) +from mayan.apps.common.serialization import yaml_dump, yaml_load + +from .utils import read_configuration_file + logger = logging.getLogger(__name__) @@ -82,10 +81,11 @@ class Namespace(object): class Setting(object): _registry = {} _cache_hash = None + _config_file_cache = None @staticmethod def deserialize_value(value): - return yaml.load(stream=value, Loader=SafeLoader) + return yaml_load(stream=value) @staticmethod def express_promises(value): @@ -101,9 +101,9 @@ class Setting(object): @staticmethod def serialize_value(value): - result = yaml.dump( + result = yaml_dump( data=Setting.express_promises(value), allow_unicode=True, - Dumper=SafeDumper + default_flow_style=False, ) # safe_dump returns bytestrings # Disregard the last 3 dots that mark the end of the YAML document @@ -128,8 +128,8 @@ class Setting(object): if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term: dictionary[setting.global_name] = Setting.express_promises(setting.value) - return yaml.dump( - data=dictionary, default_flow_style=False, Dumper=SafeDumper + return yaml_dump( + data=dictionary, default_flow_style=False ) @classmethod @@ -140,6 +140,16 @@ class Setting(object): def get_all(cls): return sorted(cls._registry.values(), key=lambda x: x.global_name) + @classmethod + def get_config_file_content(cls): + # Cache content of config file to speed up initial boot up + if not cls._config_file_cache: + cls._config_file_cache = read_configuration_file( + path=settings.CONFIGURATION_FILEPATH + ) + + return cls._config_file_cache + @classmethod def get_hash(cls): return force_text( @@ -167,13 +177,12 @@ class Setting(object): path=settings.CONFIGURATION_LAST_GOOD_FILEPATH ) - def __init__(self, namespace, global_name, default, help_text=None, is_path=False, quoted=False): + def __init__(self, namespace, global_name, default, help_text=None, is_path=False): self.global_name = global_name self.default = default self.help_text = help_text self.loaded = False self.namespace = namespace - self.quoted = quoted self.environment_variable = False namespace._settings.append(self) self.__class__._registry[global_name] = self @@ -186,7 +195,7 @@ class Setting(object): if environment_value: self.environment_variable = True try: - self.raw_value = environment_value + self.raw_value = yaml_load(stream=environment_value) except yaml.YAMLError as exception: raise type(exception)( 'Error interpreting environment variable: {} with ' @@ -195,7 +204,12 @@ class Setting(object): ) ) else: - self.raw_value = getattr(settings, self.global_name, self.default) + self.raw_value = self.get_config_file_content().get( + self.global_name, getattr( + settings, self.global_name, self.default + ) + ) + self.yaml = Setting.serialize_value(self.raw_value) self.loaded = True diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index d72548316d..ecc7067ffd 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + class SettingForm(forms.Form): value = forms.CharField( @@ -25,20 +22,8 @@ class SettingForm(forms.Form): self.fields['value'].initial = self.setting.serialized_value def clean(self): - quotes = ['"', "'"] - - if self.setting.quoted: - stripped = self.cleaned_data['value'].strip() - - if stripped[0] not in quotes or stripped[-1] not in quotes: - raise ValidationError( - _( - 'Value must be properly quoted.' - ) - ) - try: - yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['value']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/smart_settings/literals.py b/mayan/apps/smart_settings/literals.py new file mode 100644 index 0000000000..6db3f2ee9c --- /dev/null +++ b/mayan/apps/smart_settings/literals.py @@ -0,0 +1,36 @@ +from __future__ import unicode_literals + +# Default in YAML format +BOOTSTRAP_SETTING_LIST = ( + {'name': 'ALLOWED_HOSTS', 'default': "['127.0.0.1', 'localhost', '[::1]']"}, + {'name': 'APPEND_SLASH'}, + {'name': 'AUTH_PASSWORD_VALIDATORS'}, + {'name': 'COMMON_DISABLED_APPS'}, + {'name': 'COMMON_EXTRA_APPS'}, + {'name': 'DATA_UPLOAD_MAX_MEMORY_SIZE'}, + {'name': 'DATABASES'}, + {'name': 'DEBUG', 'default': 'false'}, + {'name': 'DEFAULT_FROM_EMAIL'}, + {'name': 'DISALLOWED_USER_AGENTS'}, + {'name': 'EMAIL_BACKEND'}, + {'name': 'EMAIL_HOST'}, + {'name': 'EMAIL_HOST_PASSWORD'}, + {'name': 'EMAIL_HOST_USER'}, + {'name': 'EMAIL_PORT'}, + {'name': 'EMAIL_TIMEOUT'}, + {'name': 'EMAIL_USE_SSL'}, + {'name': 'EMAIL_USE_TLS'}, + {'name': 'FILE_UPLOAD_MAX_MEMORY_SIZE'}, + {'name': 'HOME_VIEW'}, + {'name': 'INSTALLED_APPS'}, + {'name': 'INTERNAL_IPS', 'default': "['127.0.0.1']"}, + {'name': 'LANGUAGES'}, + {'name': 'LANGUAGE_CODE'}, + {'name': 'LOGIN_REDIRECT_URL', 'default': 'common:home'}, + {'name': 'LOGIN_URL', 'default': 'authentication:login_view'}, + {'name': 'LOGOUT_REDIRECT_URL', 'default': 'authentication:login_view'}, + {'name': 'STATIC_URL'}, + {'name': 'STATICFILES_STORAGE'}, + {'name': 'TIME_ZONE'}, + {'name': 'WSGI_APPLICATION'} +) diff --git a/mayan/apps/smart_settings/settings.py b/mayan/apps/smart_settings/settings.py new file mode 100644 index 0000000000..8ee2f84879 --- /dev/null +++ b/mayan/apps/smart_settings/settings.py @@ -0,0 +1,302 @@ +from __future__ import unicode_literals + +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings.classes import Namespace + +# Don't import anything on start import, we just want to make it easy +# for apps.py to activate the settings in this module. +__all__ = () +namespace = Namespace(label=_('Django'), name='django') + +setting_django_allowed_hosts = namespace.add_setting( + global_name='ALLOWED_HOSTS', default=settings.ALLOWED_HOSTS, + help_text=_( + 'A list of strings representing the host/domain names that this site ' + 'can serve. This is a security measure to prevent HTTP Host header ' + 'attacks, which are possible even under many seemingly-safe web ' + 'server configurations. Values in this list can be ' + 'fully qualified names (e.g. \'www.example.com\'), in which case ' + 'they will be matched against the request\'s Host header exactly ' + '(case-insensitive, not including port). A value beginning with a ' + 'period can be used as a subdomain wildcard: \'.example.com\' will ' + 'match example.com, www.example.com, and any other subdomain of ' + 'example.com. A value of \'*\' will match anything; in this case you ' + 'are responsible to provide your own validation of the Host header ' + '(perhaps in a middleware; if so this middleware must be listed ' + 'first in MIDDLEWARE).' + ), +) +setting_django_append_slash = namespace.add_setting( + global_name='APPEND_SLASH', default=settings.APPEND_SLASH, + help_text=_( + 'When set to True, if the request URL does not match any of the ' + 'patterns in the URLconf and it doesn\'t end in a slash, an HTTP ' + 'redirect is issued to the same URL with a slash appended. Note ' + 'that the redirect may cause any data submitted in a POST request ' + 'to be lost. The APPEND_SLASH setting is only used if ' + 'CommonMiddleware is installed (see Middleware). See also ' + 'PREPEND_WWW.' + ) +) +setting_django_auth_password_validators = namespace.add_setting( + global_name='AUTH_PASSWORD_VALIDATORS', + default=settings.AUTH_PASSWORD_VALIDATORS, + help_text=_( + 'The list of validators that are used to check the strength of ' + 'user\'s passwords.' + ) +) +setting_django_databases = namespace.add_setting( + global_name='DATABASES', default=settings.DATABASES, + help_text=_( + 'A dictionary containing the settings for all databases to be used ' + 'with Django. It is a nested dictionary whose contents map a ' + 'database alias to a dictionary containing the options for an ' + 'individual database. The DATABASES setting must configure a ' + 'default database; any number of additional databases may also ' + 'be specified.' + ), +) +setting_django_data_upload_max_memory_size = namespace.add_setting( + global_name='DATA_UPLOAD_MAX_MEMORY_SIZE', + default=settings.DATA_UPLOAD_MAX_MEMORY_SIZE, + help_text=_( + 'Default: 2621440 (i.e. 2.5 MB). The maximum size in bytes that a ' + 'request body may be before a SuspiciousOperation ' + '(RequestDataTooBig) is raised. The check is done when accessing ' + 'request.body or request.POST and is calculated against the total ' + 'request size excluding any file upload data. You can set this to ' + 'None to disable the check. Applications that are expected to ' + 'receive unusually large form posts should tune this setting. The ' + 'amount of request data is correlated to the amount of memory ' + 'needed to process the request and populate the GET and POST ' + 'dictionaries. Large requests could be used as a ' + 'denial-of-service attack vector if left unchecked. Since web ' + 'servers don\'t typically perform deep request inspection, it\'s ' + 'not possible to perform a similar check at that level. See also ' + 'FILE_UPLOAD_MAX_MEMORY_SIZE.' + ), +) +setting_django_default_from_email = namespace.add_setting( + global_name='DEFAULT_FROM_EMAIL', + default=settings.DEFAULT_FROM_EMAIL, + help_text=_( + 'Default: \'webmaster@localhost\' ' + 'Default email address to use for various automated correspondence ' + 'from the site manager(s). This doesn\'t include error messages sent ' + 'to ADMINS and MANAGERS; for that, see SERVER_EMAIL.' + ), +) +setting_django_disallowed_user_agents = namespace.add_setting( + global_name='DISALLOWED_USER_AGENTS', + default=settings.DISALLOWED_USER_AGENTS, + help_text=_( + 'Default: [] (Empty list). List of compiled regular expression ' + 'objects representing User-Agent strings that are not allowed to ' + 'visit any page, systemwide. Use this for bad robots/crawlers. ' + 'This is only used if CommonMiddleware is installed ' + '(see Middleware).' + ), +) +setting_django_email_backend = namespace.add_setting( + global_name='EMAIL_BACKEND', + default=settings.EMAIL_BACKEND, + help_text=_( + 'Default: \'django.core.mail.backends.smtp.EmailBackend\'. The ' + 'backend to use for sending emails.' + ), +) +setting_django_email_host = namespace.add_setting( + global_name='EMAIL_HOST', + default=settings.EMAIL_HOST, + help_text=_( + 'Default: \'localhost\'. The host to use for sending email.' + ), +) +setting_django_email_host_password = namespace.add_setting( + global_name='EMAIL_HOST_PASSWORD', + default=settings.EMAIL_HOST_PASSWORD, + help_text=_( + 'Default: \'\' (Empty string). Password to use for the SMTP ' + 'server defined in EMAIL_HOST. This setting is used in ' + 'conjunction with EMAIL_HOST_USER when authenticating to the ' + 'SMTP server. If either of these settings is empty, ' + 'Django won\'t attempt authentication.' + ), +) +setting_django_email_host_user = namespace.add_setting( + global_name='EMAIL_HOST_USER', + default=settings.EMAIL_HOST_USER, + help_text=_( + 'Default: \'\' (Empty string). Username to use for the SMTP ' + 'server defined in EMAIL_HOST. If empty, Django won\'t attempt ' + 'authentication.' + ), +) +setting_django_email_port = namespace.add_setting( + global_name='EMAIL_PORT', + default=settings.EMAIL_PORT, + help_text=_( + 'Default: 25. Port to use for the SMTP server defined in EMAIL_HOST.' + ), +) +setting_django_email_timeout = namespace.add_setting( + global_name='EMAIL_TIMEOUT', + default=settings.EMAIL_TIMEOUT, + help_text=_( + 'Default: None. Specifies a timeout in seconds for blocking ' + 'operations like the connection attempt.' + ), +) +setting_django_email_user_tls = namespace.add_setting( + global_name='EMAIL_USE_TLS', + default=settings.EMAIL_USE_TLS, + help_text=_( + 'Default: False. Whether to use a TLS (secure) connection when ' + 'talking to the SMTP server. This is used for explicit TLS ' + 'connections, generally on port 587. If you are experiencing ' + 'hanging connections, see the implicit TLS setting EMAIL_USE_SSL.' + ), +) +setting_django_email_user_ssl = namespace.add_setting( + global_name='EMAIL_USE_SSL', + default=settings.EMAIL_USE_SSL, + help_text=_( + 'Default: False. Whether to use an implicit TLS (secure) connection ' + 'when talking to the SMTP server. In most email documentation this ' + 'type of TLS connection is referred to as SSL. It is generally used ' + 'on port 465. If you are experiencing problems, see the explicit ' + 'TLS setting EMAIL_USE_TLS. Note that EMAIL_USE_TLS/EMAIL_USE_SSL ' + 'are mutually exclusive, so only set one of those settings to True.' + ), +) +setting_django_file_upload_max_memory_size = namespace.add_setting( + global_name='FILE_UPLOAD_MAX_MEMORY_SIZE', + default=settings.FILE_UPLOAD_MAX_MEMORY_SIZE, + help_text=_( + 'Default: 2621440 (i.e. 2.5 MB). The maximum size (in bytes) ' + 'that an upload will be before it gets streamed to the file ' + 'system. See Managing files for details. See also ' + 'DATA_UPLOAD_MAX_MEMORY_SIZE.' + ), +) +setting_django_login_url = namespace.add_setting( + global_name='LOGIN_URL', + default=settings.LOGIN_URL, + help_text=_( + 'Default: \'/accounts/login/\' The URL where requests are ' + 'redirected for login, especially when using the login_required() ' + 'decorator. This setting also accepts named URL patterns which ' + 'can be used to reduce configuration duplication since you ' + 'don\'t have to define the URL in two places (settings ' + 'and URLconf).' + ) +) +setting_django_login_redirect_url = namespace.add_setting( + global_name='LOGIN_REDIRECT_URL', + default=settings.LOGIN_REDIRECT_URL, + help_text=_( + 'Default: \'/accounts/profile/\' The URL where requests are ' + 'redirected after login when the contrib.auth.login view gets no ' + 'next parameter. This is used by the login_required() decorator, ' + 'for example. This setting also accepts named URL patterns which ' + 'can be used to reduce configuration duplication since you don\'t ' + 'have to define the URL in two places (settings and URLconf).' + ), +) +setting_django_logout_redirect_url = namespace.add_setting( + global_name='LOGOUT_REDIRECT_URL', + default=settings.LOGOUT_REDIRECT_URL, + help_text=_( + 'Default: None. The URL where requests are redirected after a user ' + 'logs out using LogoutView (if the view doesn\'t get a next_page ' + 'argument). If None, no redirect will be performed and the logout ' + 'view will be rendered. This setting also accepts named URL ' + 'patterns which can be used to reduce configuration duplication ' + 'since you don\'t have to define the URL in two places (settings ' + 'and URLconf).' + ) +) +setting_django_internal_ips = namespace.add_setting( + global_name='INTERNAL_IPS', + default=settings.INTERNAL_IPS, + help_text=_( + 'A list of IP addresses, as strings, that: Allow the debug() ' + 'context processor to add some variables to the template context. ' + 'Can use the admindocs bookmarklets even if not logged in as a ' + 'staff user. Are marked as "internal" (as opposed to "EXTERNAL") ' + 'in AdminEmailHandler emails.' + ), +) +setting_django_languages = namespace.add_setting( + global_name='LANGUAGES', + default=settings.LANGUAGES, + help_text=_( + 'A list of all available languages. The list is a list of ' + 'two-tuples in the format (language code, language name) ' + 'for example, (\'ja\', \'Japanese\'). This specifies which ' + 'languages are available for language selection. ' + 'Generally, the default value should suffice. Only set this ' + 'setting if you want to restrict language selection to a ' + 'subset of the Django-provided languages. ' + ), +) +setting_django_language_code = namespace.add_setting( + global_name='LANGUAGE_CODE', + default=settings.LANGUAGE_CODE, + help_text=_( + 'A string representing the language code for this installation. ' + 'This should be in standard language ID format. For example, U.S. ' + 'English is "en-us". It serves two purposes: If the locale ' + 'middleware isn\'t in use, it decides which translation is served ' + 'to all users. If the locale middleware is active, it provides a ' + 'fallback language in case the user\'s preferred language can\'t ' + 'be determined or is not supported by the website. It also provides ' + 'the fallback translation when a translation for a given literal ' + 'doesn\'t exist for the user\'s preferred language.' + ), +) +setting_django_static_url = namespace.add_setting( + global_name='STATIC_URL', + default=settings.STATIC_URL, + help_text=_( + 'URL to use when referring to static files located in STATIC_ROOT. ' + 'Example: "/static/" or "http://static.example.com/" ' + 'If not None, this will be used as the base path for asset ' + 'definitions (the Media class) and the staticfiles app. ' + 'It must end in a slash if set to a non-empty value.' + ), +) +setting_django_staticfiles_storage = namespace.add_setting( + global_name='STATICFILES_STORAGE', + default=settings.STATICFILES_STORAGE, + help_text=_( + 'The file storage engine to use when collecting static files with ' + 'the collectstatic management command. A ready-to-use instance of ' + 'the storage backend defined in this setting can be found at ' + 'django.contrib.staticfiles.storage.staticfiles_storage.' + ), +) +setting_django_time_zone = namespace.add_setting( + global_name='TIME_ZONE', + default=settings.TIME_ZONE, + help_text=_( + 'A string representing the time zone for this installation. ' + 'Note that this isn\'t necessarily the time zone of the server. ' + 'For example, one server may serve multiple Django-powered sites, ' + 'each with a separate time zone setting.' + ), +) +setting_django_wsgi_application = namespace.add_setting( + global_name='WSGI_APPLICATION', + default=settings.WSGI_APPLICATION, + help_text=_( + 'The full Python path of the WSGI application object that Django\'s ' + 'built-in servers (e.g. runserver) will use. The django-admin ' + 'startproject management command will create a simple wsgi.py ' + 'file with an application callable in it, and point this setting ' + 'to that application.' + ), +) diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index 6e88607f7f..4eb6839ea9 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -11,7 +11,7 @@ from mayan.apps.common.settings import setting_paginate_by from mayan.apps.common.tests import BaseTestCase from mayan.apps.storage.utils import fs_cleanup -from ..classes import Namespace, Setting +from ..classes import Setting from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE from .mixins import SmartSettingTestMixin @@ -48,8 +48,8 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): default='test value' ) # Initialize hash cache + Setting._cache_hash = None Setting.check_changed() self.assertFalse(Setting.check_changed()) test_setting.value = 'test value edited' self.assertTrue(Setting.check_changed()) - diff --git a/mayan/apps/smart_settings/urls.py b/mayan/apps/smart_settings/urls.py index 5640901c1a..56d05c5e5a 100644 --- a/mayan/apps/smart_settings/urls.py +++ b/mayan/apps/smart_settings/urls.py @@ -6,15 +6,15 @@ from .views import NamespaceDetailView, NamespaceListView, SettingEditView urlpatterns = [ url( - regex=r'^namespace/all/$', view=NamespaceListView.as_view(), + regex=r'^namespaces/$', view=NamespaceListView.as_view(), name='namespace_list' ), url( - regex=r'^namespace/(?P\w+)/$', + regex=r'^namespaces/(?P\w+)/$', view=NamespaceDetailView.as_view(), name='namespace_detail' ), url( - regex=r'^edit/(?P\w+)/$', + regex=r'^namespaces/settings/(?P\w+)/edit/$', view=SettingEditView.as_view(), name='setting_edit_view' ), ] diff --git a/mayan/apps/smart_settings/utils.py b/mayan/apps/smart_settings/utils.py new file mode 100644 index 0000000000..77ce325e0c --- /dev/null +++ b/mayan/apps/smart_settings/utils.py @@ -0,0 +1,73 @@ +from __future__ import unicode_literals + +import errno +import os + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +from .literals import BOOTSTRAP_SETTING_LIST + + +def get_default(name, fallback_default=None): + for item in BOOTSTRAP_SETTING_LIST: + if item['name'] == name: + return item.get('default', fallback_default) + + return fallback_default + + +def get_environment_variables(): + result = {} + + for setting in BOOTSTRAP_SETTING_LIST: + environment_value = os.environ.get('MAYAN_{}'.format(setting['name'])) + if environment_value: + environment_value = yaml.load(stream=environment_value, Loader=SafeLoader) + result[setting['name']] = environment_value + + return result + + +def get_environment_setting(name, fallback_default=None): + value = os.environ.get('MAYAN_{}'.format(name), get_default(name=name, fallback_default=fallback_default)) + + if value: + return yaml.load(stream=value, Loader=SafeLoader) + + +def read_configuration_file(path): + try: + with open(path) as file_object: + file_object.seek(0, os.SEEK_END) + if file_object.tell(): + file_object.seek(0) + try: + return yaml.load(stream=file_object, Loader=SafeLoader) + except yaml.YAMLError as exception: + exit( + 'Error loading configuration file: {}; {}'.format( + path, exception + ) + ) + except IOError as exception: + if exception.errno == errno.ENOENT: + return {} # No config file, return empty dictionary + else: + raise + + +def yaml_loads(data, error_message=None): + if not error_message: + error_message = 'Error loading: {}; {}' + + try: + return yaml.load(stream=data, Loader=SafeLoader) + except yaml.YAMLError as exception: + exit( + error_message.format(data, exception) + ) diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 4c07219088..fa8ba913f8 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -4,18 +4,13 @@ import imaplib import logging import poplib -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.db import models from django.utils.encoding import force_bytes from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.documents.models import Document from mayan.apps.metadata.api import set_bulk_metadata from mayan.apps.metadata.models import MetadataType @@ -142,8 +137,8 @@ class EmailBaseModel(IntervalBaseModel): with ContentFile(content=message.body, name=label) as file_object: if label == source.metadata_attachment_name: - metadata_dictionary = yaml.load( - stream=file_object.read(), Loader=SafeLoader + metadata_dictionary = yaml_load( + stream=file_object.read() ) logger.debug( 'Got metadata dictionary: %s', diff --git a/mayan/apps/sources/settings.py b/mayan/apps/sources/settings.py index 8f0156d427..aef20ad07d 100644 --- a/mayan/apps/sources/settings.py +++ b/mayan/apps/sources/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Sources'), name='sources') @@ -21,13 +21,13 @@ setting_staging_file_image_cache_storage = namespace.add_setting( default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'staging_file image files.' - ), quoted=True + ) ) setting_staging_file_image_cache_storage_arguments = namespace.add_setting( global_name='SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') - ), help_text=_( + default={ + 'location': os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') + }, help_text=_( 'Arguments to pass to the SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ) ) diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 30e23ab7c5..faa863ae21 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,23 +1,12 @@ from __future__ import unicode_literals -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_staging_file_image_cache_storage, setting_staging_file_image_cache_storage_arguments, ) -storage_staging_file_image_cache = import_string( +storage_staging_file_image_cache = get_storage_subclass( dotted_path=setting_staging_file_image_cache_storage.value -)( - **yaml.load( - stream=setting_staging_file_image_cache_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_staging_file_image_cache_storage_arguments.value) diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 9288ff57be..a49b00ce42 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -6,15 +6,11 @@ import shutil import mock from pathlib2 import Path -import yaml -try: - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeDumper from django.core import mail from django.utils.encoding import force_text +from mayan.apps.common.serialization import yaml_dump from mayan.apps.documents.models import Document from mayan.apps.documents.tests import ( GenericDocumentTestCase, TEST_COMPRESSED_DOCUMENT_PATH, @@ -213,8 +209,8 @@ class EmailBaseTestCase(GenericDocumentTestCase): metadata_type=test_metadata_type_2 ) - test_metadata_yaml = yaml.dump( - Dumper=SafeDumper, data={ + test_metadata_yaml = yaml_dump( + data={ test_metadata_type_1.name: TEST_METADATA_VALUE_1, test_metadata_type_2.name: TEST_METADATA_VALUE_2, } diff --git a/mayan/apps/storage/classes.py b/mayan/apps/storage/classes.py new file mode 100644 index 0000000000..0c4bdd2cd5 --- /dev/null +++ b/mayan/apps/storage/classes.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + + +class FakeStorageSubclass(object): + """ + Placeholder class to allow serializing the real storage subclass to + support migrations. + """ + def __eq__(self, other): + return True diff --git a/mayan/apps/storage/settings.py b/mayan/apps/storage/settings.py index e17acbd76c..362aa9e702 100644 --- a/mayan/apps/storage/settings.py +++ b/mayan/apps/storage/settings.py @@ -4,7 +4,7 @@ import tempfile from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Storage'), name='storage') diff --git a/mayan/apps/storage/utils.py b/mayan/apps/storage/utils.py index 90eab5bae5..5cc27d0d82 100644 --- a/mayan/apps/storage/utils.py +++ b/mayan/apps/storage/utils.py @@ -5,6 +5,8 @@ import os import shutil import tempfile +from django.utils.module_loading import import_string + from .settings import setting_temporary_directory logger = logging.getLogger(__name__) @@ -39,6 +41,27 @@ def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): raise +def get_storage_subclass(dotted_path): + """ + Import a storage class and return a subclass that will always return eq + True to avoid creating a new migration when for runtime storage class + changes. + """ + imported_storage_class = import_string(dotted_path=dotted_path) + + class StorageSubclass(imported_storage_class): + def __init__(self, *args, **kwargs): + return super(StorageSubclass, self).__init__(*args, **kwargs) + + def __eq__(self, other): + return True + + def deconstruct(self): + return ('mayan.apps.storage.classes.FakeStorageSubclass', (), {}) + + return StorageSubclass + + def mkdtemp(*args, **kwargs): kwargs.update({'dir': setting_temporary_directory.value}) return tempfile.mkdtemp(*args, **kwargs) diff --git a/mayan/apps/tags/tests/test_actions.py b/mayan/apps/tags/tests/test_actions.py index 0aa8c55078..ec0fe3caa5 100644 --- a/mayan/apps/tags/tests/test_actions.py +++ b/mayan/apps/tags/tests/test_actions.py @@ -39,7 +39,7 @@ class TagActionViewTestCase(WorkflowTestMixin, GenericViewTestCase): self._create_test_workflow_state() response = self.get( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.tags.workflow_actions.AttachTagAction' @@ -53,7 +53,7 @@ class TagActionViewTestCase(WorkflowTestMixin, GenericViewTestCase): self._create_test_workflow_state() response = self.get( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.tags.workflow_actions.RemoveTagAction' diff --git a/mayan/apps/task_manager/apps.py b/mayan/apps/task_manager/apps.py index 00996c6528..781632b927 100644 --- a/mayan/apps/task_manager/apps.py +++ b/mayan/apps/task_manager/apps.py @@ -13,6 +13,7 @@ from .links import ( link_queue_scheduled_task_list, link_queue_reserved_task_list, link_task_manager ) +from .settings import * # NOQA class TaskManagerApp(MayanAppConfig): diff --git a/mayan/apps/task_manager/settings.py b/mayan/apps/task_manager/settings.py new file mode 100644 index 0000000000..9074439afe --- /dev/null +++ b/mayan/apps/task_manager/settings.py @@ -0,0 +1,29 @@ +from __future__ import unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings.classes import Namespace + +# Don't import anything on start import, we just want to make it easy +# for apps.py to activate the settings in this module. +__all__ = () +namespace = Namespace(label=_('Celery'), name='celery') + +setting_celery_broker_url = namespace.add_setting( + global_name='BROKER_URL', default=None, + help_text=_( + 'Default: "amqp://". Default broker URL. This must be a URL in ' + 'the form of: transport://userid:password@hostname:port/virtual_host ' + 'Only the scheme part (transport://) is required, the rest is ' + 'optional, and defaults to the specific transports default values.' + ), +) +setting_celery_result_backend = namespace.add_setting( + global_name='CELERY_RESULT_BACKEND', default=None, + help_text=_( + 'Default: No result backend enabled by default. The backend used ' + 'to store task results (tombstones). Refer to ' + 'http://docs.celeryproject.org/en/v4.1.0/userguide/configuration.' + 'html#result-backend' + ) +) diff --git a/mayan/apps/user_management/tests/mixins.py b/mayan/apps/user_management/tests/mixins.py index 548df197e8..a37792fd55 100644 --- a/mayan/apps/user_management/tests/mixins.py +++ b/mayan/apps/user_management/tests/mixins.py @@ -249,11 +249,19 @@ class UserTestMixin(object): self.test_superuser.cleartext_password = TEST_USER_PASSWORD def _create_test_user(self): + total_test_users = len(self.test_users) + username = '{}_{}'.format(TEST_USER_USERNAME, total_test_users) + self.test_user = get_user_model().objects.create_user( - username=TEST_USER_USERNAME, email=TEST_USER_EMAIL, + username=username, email=TEST_USER_EMAIL, password=TEST_USER_PASSWORD ) self.test_user.cleartext_password = TEST_USER_PASSWORD + self.test_users.append(self.test_user) + + def setUp(self): + super(UserTestMixin, self).setUp() + self.test_users = [] class UserViewTestMixin(object): diff --git a/mayan/conf.py b/mayan/conf.py index 212322bd5d..898ccdd7da 100644 --- a/mayan/conf.py +++ b/mayan/conf.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(name='mayan', label=_('Mayan')) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index f2f3888ac2..e3bd63c66b 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -17,15 +17,15 @@ import sys from django.core.exceptions import ImproperlyConfigured from django.utils.translation import ugettext_lazy as _ -import environ +from mayan.apps.smart_settings.literals import BOOTSTRAP_SETTING_LIST +from mayan.apps.smart_settings.utils import ( + get_environment_setting, get_environment_variables, read_configuration_file +) from .literals import ( CONFIGURATION_FILENAME, CONFIGURATION_LAST_GOOD_FILENAME, DEFAULT_SECRET_KEY, SECRET_KEY_FILENAME, SYSTEM_DIR ) -from .utils import yaml_loads, read_configuration_file - -env = environ.Env() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -34,8 +34,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ -MEDIA_ROOT = os.environ.get( - 'MAYAN_MEDIA_ROOT', os.path.join(BASE_DIR, 'media') +MEDIA_ROOT = get_environment_setting( + name='MAYAN_MEDIA_ROOT', fallback_default=os.path.join(BASE_DIR, 'media') ) # SECURITY WARNING: keep the secret key used in production secret! @@ -50,13 +50,9 @@ else: SECRET_KEY = DEFAULT_SECRET_KEY # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = env.bool('MAYAN_DEBUG', default=False) +DEBUG = get_environment_setting(name='DEBUG') -ALLOWED_HOSTS = yaml_loads( - env( - 'MAYAN_ALLOWED_HOSTS', default="['127.0.0.1', 'localhost', '[::1]']" - ) -) +ALLOWED_HOSTS = get_environment_setting(name='ALLOWED_HOSTS') # Application definition @@ -259,12 +255,10 @@ TEST_RUNNER = 'mayan.apps.common.tests.runner.MayanTestRunner' # --------- Django ------------------- -LOGIN_URL = env('MAYAN_LOGIN_URL', default='authentication:login_view') -LOGIN_REDIRECT_URL = env('MAYAN_LOGIN_REDIRECT_URL', default='common:root') -LOGOUT_REDIRECT_URL = env( - 'MAYAN_LOGOUT_REDIRECT_URL', default='authentication:login_view' -) -INTERNAL_IPS = ('127.0.0.1',) +LOGIN_URL = get_environment_setting(name='LOGIN_URL') +LOGIN_REDIRECT_URL = get_environment_setting(name='LOGIN_REDIRECT_URL') +LOGOUT_REDIRECT_URL = get_environment_setting(name='LOGOUT_REDIRECT_URL') +INTERNAL_IPS = get_environment_setting(name='INTERNAL_IPS') # ---------- Django REST framework ----------- @@ -327,51 +321,43 @@ AJAX_REDIRECT_CODE = 278 # ----- Celery ----- -BROKER_URL = os.environ.get('MAYAN_BROKER_URL') -CELERY_ALWAYS_EAGER = env.bool('MAYAN_CELERY_ALWAYS_EAGER', default=True) -CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND') +BROKER_URL = get_environment_setting(name='BROKER_URL') +CELERY_ALWAYS_EAGER = get_environment_setting(name='CELERY_ALWAYS_EAGER') +CELERY_RESULT_BACKEND = get_environment_setting(name='CELERY_RESULT_BACKEND') # ----- Database ----- -environment_database_engine = os.environ.get('MAYAN_DATABASE_ENGINE') - -if environment_database_engine: - environment_database_conn_max_age = os.environ.get('MAYAN_DATABASE_CONN_MAX_AGE', 0) - if environment_database_conn_max_age: - environment_database_conn_max_age = int(environment_database_conn_max_age) - - DATABASES = { - 'default': { - 'ENGINE': environment_database_engine, - 'NAME': os.environ['MAYAN_DATABASE_NAME'], - 'USER': os.environ['MAYAN_DATABASE_USER'], - 'PASSWORD': os.environ['MAYAN_DATABASE_PASSWORD'], - 'HOST': os.environ.get('MAYAN_DATABASE_HOST', None), - 'PORT': os.environ.get('MAYAN_DATABASE_PORT', None), - 'CONN_MAX_AGE': environment_database_conn_max_age, - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), - } - } - +} BASE_INSTALLED_APPS = INSTALLED_APPS COMMON_EXTRA_APPS = () COMMON_DISABLED_APPS = () -CONFIGURATION_FILEPATH = os.path.join(MEDIA_ROOT, CONFIGURATION_FILENAME) -CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join( - MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME +CONFIGURATION_FILEPATH = get_environment_setting( + name='CONFIGURATION_FILEPATH', fallback_default=os.path.join( + MEDIA_ROOT, CONFIGURATION_FILENAME + ) +) + +CONFIGURATION_LAST_GOOD_FILEPATH = get_environment_setting( + name='CONFIGURATION_LAST_GOOD_FILEPATH', fallback_default=os.path.join( + MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME + ) ) if 'revertsettings' not in sys.argv: - result = read_configuration_file(CONFIGURATION_FILEPATH) - if result: - globals().update(result) + configuration_result = read_configuration_file(CONFIGURATION_FILEPATH) + environment_result = get_environment_variables() + + for setting in BOOTSTRAP_SETTING_LIST: + if setting['name'] in configuration_result: + globals().update({setting['name']: configuration_result[setting['name']]}) + elif setting['name'] in environment_result: + globals().update({setting['name']: environment_result[setting['name']]}) for app in INSTALLED_APPS: diff --git a/mayan/settings/utils.py b/mayan/settings/utils.py deleted file mode 100644 index bb2a8da489..0000000000 --- a/mayan/settings/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import unicode_literals - -import errno -import os - -import yaml - - -def read_configuration_file(path): - try: - with open(path) as file_object: - file_object.seek(0, os.SEEK_END) - if file_object.tell(): - file_object.seek(0) - try: - return yaml.safe_load(file_object) - except yaml.YAMLError as exception: - exit( - 'Error loading configuration file: {}; {}'.format( - path, exception - ) - ) - except IOError as exception: - if exception.errno == errno.ENOENT: - pass - else: - raise - - -def yaml_loads(data, error_message=None): - if not error_message: - error_message = 'Error loading: {}; {}' - - try: - return yaml.safe_load(data) - except yaml.YAMLError as exception: - exit( - error_message.format(data, exception) - ) diff --git a/removals.txt b/removals.txt index 6a40aee535..2163f9ee5d 100644 --- a/removals.txt +++ b/removals.txt @@ -1,6 +1,7 @@ # Packages to be remove during upgrades cssmin django-autoadmin +django-environ django-suit django-compressor django-filetransfers