Added python-magic for smarter MIME type detection and a new document field: file_mime_encoding

modules/python_magic/testdata/test.pdf

@@ -0,0 +1,199 @@
+%PDF-1.2
+7 0 obj
+[5 0 R/XYZ 111.6 757.86]
+endobj
+13 0 obj
+<<
+/Title(About this document)
+/A<<
+/S/GoTo
+/D(subsection.1.1)
+>>
+/Parent 12 0 R
+/Next 14 0 R
+>>
+endobj
+15 0 obj
+<<
+/Title(Compiling with GHC)
+/A<<
+/S/GoTo
+/D(subsubsection.1.2.1)
+>>
+/Parent 14 0 R
+/Next 16 0 R
+>>
+endobj
+16 0 obj
+<<
+/Title(Compiling with Hugs)
+/A<<
+/S/GoTo
+/D(subsubsection.1.2.2)
+>>
+/Parent 14 0 R
+/Prev 15 0 R
+>>
+endobj
+14 0 obj
+<<
+/Title(Compatibility)
+/A<<
+/S/GoTo
+/D(subsection.1.2)
+>>
+/Parent 12 0 R
+/Prev 13 0 R
+/First 15 0 R
+/Last 16 0 R
+/Count -2
+/Next 17 0 R
+>>
+endobj
+17 0 obj
+<<
+/Title(Reporting bugs)
+/A<<
+/S/GoTo
+/D(subsection.1.3)
+>>
+/Parent 12 0 R
+/Prev 14 0 R
+/Next 18 0 R
+>>
+endobj
+18 0 obj
+<<
+/Title(History)
+/A<<
+/S/GoTo
+/D(subsection.1.4)
+>>
+/Parent 12 0 R
+/Prev 17 0 R
+/Next 19 0 R
+>>
+endobj
+19 0 obj
+<<
+/Title(License)
+/A<<
+/S/GoTo
+/D(subsection.1.5)
+>>
+/Parent 12 0 R
+/Prev 18 0 R
+>>
+endobj
+12 0 obj
+<<
+/Title(Introduction)
+/A<<
+/S/GoTo
+/D(section.1)
+>>
+/Parent 11 0 R
+/First 13 0 R
+/Last 19 0 R
+/Count -5
+/Next 20 0 R
+>>
+endobj
+21 0 obj
+<<
+/Title(Running a parser)
+/A<<
+/S/GoTo
+/D(subsection.2.1)
+>>
+/Parent 20 0 R
+/Next 22 0 R
+>>
+endobj
+22 0 obj
+<<
+/Title(Sequence and choice)
+/A<<
+/S/GoTo
+/D(subsection.2.2)
+>>
+/Parent 20 0 R
+/Prev 21 0 R
+/Next 23 0 R
+>>
+endobj
+23 0 obj
+<<
+/Title(Predictive parsers)
+/A<<
+/S/GoTo
+/D(subsection.2.3)
+>>
+/Parent 20 0 R
+/Prev 22 0 R
+/Next 24 0 R
+>>
+endobj
+24 0 obj
+<<
+/Title(Adding semantics)
+/A<<
+/S/GoTo
+/D(subsection.2.4)
+>>
+/Parent 20 0 R
+/Prev 23 0 R
+/Next 25 0 R
+>>
+endobj
+25 0 obj
+<<
+/Title(Sequences and seperators)
+/A<<
+/S/GoTo
+/D(subsection.2.5)
+>>
+/Parent 20 0 R
+/Prev 24 0 R
+/Next 26 0 R
+>>
+endobj
+26 0 obj
+<<
+/Title(Improving error messages)
+/A<<
+/S/GoTo
+/D(subsection.2.6)
+>>
+/Parent 20 0 R
+/Prev 25 0 R
+/Next 27 0 R
+>>
+endobj
+27 0 obj
+<<
+/Title(Expressions)
+/A<<
+/S/GoTo
+/D(subsection.2.7)
+>>
+/Parent 20 0 R
+/Prev 26 0 R
+/Next 28 0 R
+>>
+endobj
+28 0 obj
+<<
+/Title(Lexical analysis)
+/A<<
+/S/GoTo
+/D(subsection.2.8)
+>>
+/Parent 20 0 R
+/Prev 27 0 R
+/Next 29 0 R
+>>
+endobj
+30 0 obj
+<<
+/Title(Lexeme parsers

modules/python_magic/testdata/text-iso8859-1.txt

@@ -0,0 +1,2 @@
+This is a web page encoded in iso-8859-1
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

modules/python_magic/testdata/text.txt

@@ -0,0 +1,2 @@
+Hello, World!
+