Support Google Font dependencies

Allow downloading fonts from Google Font at install time.
Closes GitLab issue #595, thanks to Martin (@efelon) for the
report. Closes re-opened GitLab issue #343.
Remove included Lato font.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

.gitignore

@@ -30,4 +30,5 @@ static_collected/
 /venv/
 /venv3/
 /whoosh_index/
+google_fonts/
 node_modules/

HISTORY.rst

@@ -278,6 +278,12 @@
 * Add note about the new preparestatic command.
 * Add no-result template for workflow instance detail view.
 * Update HTTP workflow action to new requests API.
+* Remove the included Lato font. The font is now downloaded
+  at install time.
+* Add support for Google Fonts dependencies.
+* Add support for patchin dependency files using rewriting rules.
+
+
 
 3.1.11 (2019-04-XX)
 ===================

docs/releases/3.2.rst

@@ -706,7 +706,10 @@ Other changes
   Improve responsive settings. Redirect to the current view after queueing.
 - Split document type retention policies into it own view.
 - Place deletion policies units before periods for clarity.
-
+- Remove the included Lato font. The font is now downloaded
+  at install time.
+- Add support for Google Fonts dependencies.
+- Add support for patchin dependency files using rewriting rules.
 
 Removals
 --------
@@ -854,5 +857,6 @@ Bugs fixed or issues closed
 - :gitlab-issue:`563` Recursive Watch Folder
 - :gitlab-issue:`579` Untranslated items
 - :gitlab-issue:`589` Document {{ link }} send via Email contains example.com as domain
+- :gitlab-issue:`595` Remove dependency to fonts.googleapis.com
 
 .. _PyPI: https://pypi.python.org/pypi/mayan-edms/

mayan/apps/appearance/dependencies.py

@@ -2,15 +2,31 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.dependencies.classes import JavaScriptDependency
+from mayan.apps.dependencies.classes import (
+    GoogleFontDependency, JavaScriptDependency
+)
 
+GoogleFontDependency(
+    label=_('Lato font'), module=__name__, name='lato',
+    url='https://fonts.googleapis.com/css?family=Lato:400,700,400italic'
+)
 JavaScriptDependency(
     label=_('Bootstrap'), module=__name__, name='bootstrap',
     version_string='=3.4.1'
 )
 JavaScriptDependency(
     label=_('Bootswatch'), module=__name__, name='bootswatch',
-    version_string='=3.4.1'
+    replace_list = [
+        {
+            'filename_pattern': '*.css',
+            'content_patterns': [
+                {
+                    'search': '"https://fonts.googleapis.com/css?family=Lato:400,700,400italic"',
+                    'replace': '../../../google_fonts/lato/import.css',
+                }
+            ]
+        }
+    ], version_string='=3.4.1'
 )
 JavaScriptDependency(
     label=_('Fancybox'), module=__name__, name='@fancyapps/fancybox',

mayan/apps/appearance/static/appearance/css/base.css

@@ -11,44 +11,6 @@
         url('../fonts/IM_Fell_English_SC.ttf') format('truetype');
 }
 
-/* Flatly fonts */
-@font-face {
-    font-family: 'Lato';
-    font-style: normal;
-    font-weight: 400;
-    src:
-        local('Lato Regular'),
-        local('Lato-Regular'),
-        url('../fonts/Lato_400.eot?#iefix') format('embedded-opentype'),
-        url('../fonts/Lato_400.woff') format('woff'),
-        url('../fonts/Lato_400.svg#Lato') format('svg'),
-        url('../fonts/Lato_400.ttf') format('truetype');
-}
-@font-face {
-    font-family: 'Lato';
-    font-style: normal;
-    font-weight: 700;
-    src:
-        local('Lato Bold'),
-        local('Lato-Bold'),
-        url('../fonts/Lato_700.eot?#iefix') format('embedded-opentype'),
-        url('../fonts/Lato_700.woff') format('woff'),
-        url('../fonts/Lato_700.svg#Lato') format('svg'),
-        url('../fonts/Lato_700.ttf') format('truetype');
-}
-@font-face {
-    font-family: 'Lato';
-    font-style: italic;
-    font-weight: 400;
-    src:
-        local('Lato Italic'),
-        local('Lato-Italic'),
-        url('../fonts/Lato_400italic.eot?#iefix') format('embedded-opentype'),
-        url('../fonts/Lato_400italic.woff') format('woff'),
-        url('../fonts/Lato_400italic.svg#Lato') format('svg'),
-        url('../fonts/Lato_400italic.ttf') format('truetype');
-}
-
 body {
     padding-top: 70px;
 }

mayan/apps/dependencies/apps.py

@@ -22,7 +22,7 @@ class DependenciesApp(MayanAppConfig):
     app_namespace = 'dependencies'
     app_url = 'dependencies'
     has_rest_api = False
-    has_tests = False
+    has_tests = True
     name = 'mayan.apps.dependencies'
     verbose_name = _('Dependencies')
 

mayan/apps/dependencies/classes.py

@@ -1,5 +1,6 @@
 from __future__ import print_function, unicode_literals
 
+import fileinput
 import json
 import pkg_resources
 import shutil
@@ -36,6 +37,10 @@ class PyPIRespository(Provider):
     url = 'https://pypi.org/'
 
 
+class GoogleFontsProvider(Provider):
+    url = 'https://fonts.googleapis.com/'
+
+
 class NPMRegistryRespository(Provider):
     url = 'http://registry.npmjs.com'
 
@@ -159,7 +164,7 @@ class Dependency(object):
 
     @classmethod
     def check_all(cls):
-        template = '{:<35}{:<10} {:<15} {:<20} {:<15} {:<30} {:<10}'
+        template = '{:<35}{:<11} {:<15} {:<20} {:<15} {:<30} {:<10}'
 
         print('\n  ', end='')
         print(
@@ -249,7 +254,7 @@ class Dependency(object):
     def __init__(
         self, name, app_label=None, copyright_text=None, help_text=None,
         environment=environment_production, label=None, module=None,
-        version_string=None
+        replace_list=None, version_string=None
     ):
         self._app_label = app_label
         self.copyright_text = copyright_text
@@ -259,6 +264,7 @@ class Dependency(object):
         self.module = module
         self.name = name
         self.package_metadata = None
+        self.replace_list = replace_list
         self.repository = self.provider_class()
         self.version_string = version_string
 
@@ -307,13 +313,21 @@ class Dependency(object):
                 print(_('Complete.'))
                 sys.stdout.flush()
         else:
-            self._install()
+            if self.replace_list:
+                self.patch_files()
+                print(_('Complete.'))
+                sys.stdout.flush()
+
+            self.patch_files()
             print(_('Complete.'))
             sys.stdout.flush()
 
     def _install(self):
         raise NotImplementedError
 
+    def __repr__(self):
+        return '<{}: {}>'.format(self.__class__.__name__, self.name)
+
     def check(self):
         """
         Returns the version found or an exception
@@ -373,6 +387,39 @@ class Dependency(object):
     def get_version_string(self):
         return self.version_string or _('Not specified')
 
+    def patch_files(self, path=None, replace_list=None):
+        """
+        Search and replace content from a list of file based on a pattern
+        replace_list[
+            {
+                'filename_pattern': '*.css',
+                'content_patterns': [
+                    {
+                        'search': '',
+                        'replace': '',
+                    }
+                ]
+            }
+        ]
+        """
+        print(_('Patching files... '), end='')
+        sys.stdout.flush()
+
+        if not path:
+            path = self.get_install_path()
+
+        if not replace_list:
+            replace_list = self.replace_list
+
+        path_object = Path(path)
+        for replace_entry in replace_list or []:
+            for path_entry in path_object.glob('**/{}'.format(replace_entry['filename_pattern'])):
+                if path_entry.is_file():
+                    with fileinput.FileInput(path_entry, inplace=True, backup='.bck') as fo:
+                        for line in fo:
+                            for pattern in replace_entry['content_patterns']:
+                                print(line.replace(pattern['search'], pattern['replace']), end='')
+
     def verify(self):
         """
         Verify the integrity of the dependency
@@ -459,13 +506,15 @@ class JavaScriptDependency(Dependency):
                 )
                 dependency.install(include_dependencies=False)
 
-    def extract(self):
+    def extract(self, replace_list=None):
         temporary_directory = mkdtemp()
         path_compressed_file = self.get_tar_file_path()
 
         with tarfile.open(name=force_text(path_compressed_file), mode='r') as file_object:
             file_object.extractall(path=temporary_directory)
 
+        self.patch_files(path=temporary_directory, replace_list=replace_list)
+
         path_install = self.get_install_path()
 
         # Clear the installation path of previous content
@@ -651,6 +700,88 @@ class PythonDependency(Dependency):
             return super(PythonDependency, self).get_copyright()
 
 
+class GoogleFontDependency(Dependency):
+    class_name = 'google_font'
+    class_name_help_text = _(
+        'Fonts downloaded from fonts.googleapis.com.'
+    )
+    class_name_verbose_name = _('Google font')
+    provider_class = GoogleFontsProvider
+    user_agents = {
+        'woff2': 'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0',
+        'woff': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
+        'ttf': 'Mozilla/5.0 (Linux; U; Android 2.2; en-us; DROID2 GLOBAL Build/S273) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.url = kwargs.pop('url')
+        self.static_folder = kwargs.pop('static_folder', None)
+        super(GoogleFontDependency, self).__init__(*args, **kwargs)
+
+    def _check(self):
+        return self.get_install_path().exists()
+
+    def _install(self):
+        print(_('Downloading... '), end='')
+        sys.stdout.flush()
+        self.download()
+        print(_('Extracting... '), end='')
+        sys.stdout.flush()
+        self.extract()
+
+    def download(self):
+        self.path_cache = Path(mkdtemp())
+        # Use .css to keep the same ContentType, otherwise the webserver
+        # will use the generic octet and the browser will ignore the import
+        # https://www.w3.org/TR/2013/CR-css-cascade-3-20131003/#content-type
+        self.path_import_file = self.path_cache / 'import.css'
+
+        self.font_files = []
+
+        with open(self.path_import_file, mode='w') as file_object:
+            for agent_name, agent_string in self.user_agents.items():
+                import_file = force_text(
+                    requests.get(
+                        self.url, headers={
+                            'User-Agent': agent_string
+                        }
+                    ).content
+                )
+
+                for line in import_file.split('\n'):
+                    if 'url' in line:
+                        font_url = line.split(' ')[-2][4:-1]
+                        url = furl(force_text(font_url))
+                        font_filename = url.path.segments[-1]
+
+                        with open(self.path_cache / font_filename, mode='wb') as font_file_object:
+                            with requests.get(font_url, stream=True) as response:
+                                shutil.copyfileobj(fsrc=response.raw, fdst=font_file_object)
+
+                        line = line.replace(font_url, font_filename)
+
+                    file_object.write(line)
+
+    def extract(self, replace_list=None):
+        path_install = self.get_install_path()
+
+        # Clear the installation path of previous content
+        shutil.rmtree(path=force_text(path_install), ignore_errors=True)
+
+        shutil.copytree(
+            force_text(self.path_cache), force_text(path_install)
+        )
+        shutil.rmtree(force_text(self.path_cache), ignore_errors=True)
+
+    def get_install_path(self):
+        app = apps.get_app_config(app_label=self.app_label)
+        result = Path(
+            app.path, 'static', self.static_folder or app.label,
+            'google_fonts', self.name
+        )
+        return result
+
+
 DependencyGroup(
     attribute_name='app_label', label=_('Declared in app'), help_text=_(
         'Show depedencies by the app that declared them.'

mayan/apps/dependencies/management/commands/installjavascript.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 from django.core import management
 from django.utils.translation import ugettext_lazy as _
 
-from ...classes import JavaScriptDependency
+from ...classes import GoogleFontDependency, JavaScriptDependency
 
 
 class Command(management.BaseCommand):
@@ -24,3 +24,7 @@ class Command(management.BaseCommand):
             app_label=options['app'], force=options['force'],
             subclass_only=True
         )
+        GoogleFontDependency.install_multiple(
+            app_label=options['app'], force=options['force'],
+            subclass_only=True
+        )

mayan/apps/dependencies/tests/test_classes.py

@@ -0,0 +1,53 @@
+from __future__ import print_function, unicode_literals
+
+from pathlib2 import Path
+import shutil
+
+from mayan.apps.common.tests import BaseTestCase
+from mayan.apps.storage.utils import mkdtemp
+
+from ..classes import Dependency, Provider
+
+
+class TestProvider(Provider):
+    """Test provider"""
+
+
+class TestDependency(Dependency):
+    provider_class = TestProvider
+
+
+class DependencyClassTestCase(BaseTestCase):
+    def test_file_patching(self):
+        test_replace_text = 'replaced_text'
+
+        temporary_directory = mkdtemp()
+        path_temporary_directory = Path(temporary_directory)
+        path_test_file = path_temporary_directory / 'test_file.css'
+
+        with open(path_test_file, mode='w') as file_object:
+            file_object.write(
+                '@import url("https://fonts.googleapis.com/css?family=Lato:400,700,400italic");'
+            )
+
+        dependency = TestDependency(name='test_dependency', module=__name__)
+        replace_list = [
+            {
+                'filename_pattern': '*',
+                'content_patterns': [
+                    {
+                        'search': '"https://fonts.googleapis.com/css?family=Lato:400,700,400italic"',
+                        'replace': test_replace_text,
+                    }
+                ]
+            }
+        ]
+
+        dependency.patch_files(path=temporary_directory, replace_list=replace_list)
+
+        with open(path_test_file, mode='r') as file_object:
+            final_text = file_object.read()
+
+        shutil.rmtree(temporary_directory, ignore_errors=True)
+
+        self.assertEqual(final_text, '@import url({});'.format(test_replace_text))