aboutcode-org · pombredanne · Oct 3, 2020 · Oct 3, 2020 · Oct 3, 2020 · Oct 3, 2020
diff --git a/.gitignore b/.gitignore
@@ -67,3 +67,16 @@ pyvenv.cfg
 /.pytest_cache/
 lib64
 tcl
+
+
+__pycache__
+
+
+
+# test files that should not be ignored
+!/tests/typecode/data/contenttype/compiled/java/*
+!/tests/typecode/data/contenttype/compiled/python/*
+!/tests/typecode/data/contenttype/compiled/flash/*
+!/tests/typecode/data/contenttype/compiled/linux/*
+!/tests/typecode/data/contenttype/compiled/win/*
+!/tests/typecode/data/contenttype/package/*
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,7 +30,7 @@ norecursedirs = [
    "Scripts",
    "thirdparty",
    "tmp",
-   "tests/data",
+   "tests/typecode/data",
    ".eggs"
 ]
 

diff --git a/src/typecode/contenttype.py b/src/typecode/contenttype.py
@@ -25,6 +25,7 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+from collections import OrderedDict
 import contextlib
 import io
 import os
@@ -57,7 +58,6 @@
 from typecode.pygments_lexers import get_lexer_for_filename
 from typecode.pygments_lexers import guess_lexer
 
-
 """
 Utilities to detect and report the type of a file or path based on its name,
 extension and mostly its content.
@@ -82,15 +82,13 @@ def logger_debug(*args):
     def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, string_types) and a or repr(a) for a in args))
 
-
 data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
 # Python mimetypes path setup using Apache mimetypes DB
 os.environ['XDG_DATA_DIRS'] = os.path.join(data_dir, 'apache')
 os.environ['XDG_DATA_HOME'] = os.environ['XDG_DATA_DIRS']
 APACHE_MIME_TYPES = os.path.join(data_dir, 'apache', 'mime.types')
 
-
 # Ensure that all dates are UTC, especially for fine free file.
 os.environ['TZ'] = 'UTC'
 
@@ -108,7 +106,6 @@ def as_bytes(items):
 ELF_UNKNOWN = 'unknown'
 elf_types = (ELF_EXE, ELF_SHARED, ELF_RELOC,)
 
-
 # TODO:
 # http://svn.zope.org/z3c.mimetype/trunk/?pathrev=103648
 # http://svn.zope.org/z3c.sharedmimeinfo/trunk/TODO.txt?revision=103668&view=markup
@@ -172,6 +169,55 @@ class Type(object):
         '_contains_text',
     )
 
+    # FIXME: we should use an introspectable attrs class instead
+    # ATTENTION: keep this in sync with sloats and properties
+    text_attributes = [
+        'filetype_file',
+        'mimetype_file',
+        'mimetype_python',
+        'filetype_pygment',
+        'elf_type',
+        'programming_language',
+    ]
+
+    exportable_attributes = text_attributes + [
+        'is_file',
+        'is_dir',
+        'is_regular',
+        'is_special',
+        'date',
+        'is_link',
+        'is_broken_link',
+        'link_target',
+        'size',
+        'is_pdf_with_text',
+        'is_text',
+        'is_text_with_long_lines',
+        'is_compact_js',
+        'is_js_map',
+        'is_binary',
+        'is_data',
+        'is_archive',
+        'contains_text',
+        'is_compressed',
+        'is_c_source',
+        'is_c_source',
+        'is_elf',
+        'is_elf',
+        'is_filesystem',
+        'is_java_class',
+        'is_java_source',
+        'is_media',
+        'is_media_with_meta',
+        'is_office_doc',
+        'is_package',
+        'is_pdf',
+        'is_script',
+        'is_source',
+        'is_stripped_elf',
+        'is_winexe',
+    ]
+
     def __init__(self, location):
         if (not location
             or (not os.path.exists(location)
@@ -215,6 +261,15 @@ def __repr__(self):
                 % (self.filetype_file, self.mimetype_file,
                    self.filetype_pygment, self.mimetype_python))
 
+    def to_dict(self, include_date=True):
+        """
+        Return a mapping of attributes.
+        """
+        nv = ((n, getattr(self, n)) for n in self.exportable_attributes)
+        if not include_date:
+            nv = ((n,v) for n,v in nv if n != 'date')
+        return OrderedDict(nv)
+
     @property
     def size(self):
         """

diff --git a/...ypecode/data/contenttype/archive/a.tar.gz → tests/typecode/data/extractible/a.tar.gz b/...ypecode/data/contenttype/archive/a.tar.gz → tests/typecode/data/extractible/a.tar.gz
diff --git a/tests/typecode/data/extractible/crashing-squashfs b/tests/typecode/data/extractible/crashing-squashfs
diff --git a/...pecode/data/contenttype/archive/dbase.fdt → tests/typecode/data/extractible/dbase.fdt b/...pecode/data/contenttype/archive/dbase.fdt → tests/typecode/data/extractible/dbase.fdt
diff --git a/...s/typecode/data/contenttype/archive/e.tar → tests/typecode/data/extractible/e.tar b/...s/typecode/data/contenttype/archive/e.tar → tests/typecode/data/extractible/e.tar
diff --git a/...pecode/data/contenttype/archive/e.tar.bz2 → tests/typecode/data/extractible/e.tar.bz2 b/...pecode/data/contenttype/archive/e.tar.bz2 → tests/typecode/data/extractible/e.tar.bz2
diff --git a/...ypecode/data/contenttype/archive/e.tar.gz → tests/typecode/data/extractible/e.tar.gz b/...ypecode/data/contenttype/archive/e.tar.gz → tests/typecode/data/extractible/e.tar.gz
diff --git a/...a/contenttype/archive/file_4.26-1.diff.gz → ...code/data/extractible/file_4.26-1.diff.gz b/...a/contenttype/archive/file_4.26-1.diff.gz → ...code/data/extractible/file_4.26-1.diff.gz
diff --git a/.../data/contenttype/archive/posixnotgnu.tar → ...typecode/data/extractible/posixnotgnu.tar b/.../data/contenttype/archive/posixnotgnu.tar → ...typecode/data/extractible/posixnotgnu.tar
diff --git a/tests/typecode/data/extractible/sqfs-gz.sqs b/tests/typecode/data/extractible/sqfs-gz.sqs
diff --git a/tests/typecode/data/extractible/sqfs-lzo.sqs b/tests/typecode/data/extractible/sqfs-lzo.sqs
diff --git a/tests/typecode/data/extractible/sqfs-xz.sqs b/tests/typecode/data/extractible/sqfs-xz.sqs
diff --git a/...de/data/contenttype/archive/test.tar.lzma → ...s/typecode/data/extractible/test.tar.lzma b/...de/data/contenttype/archive/test.tar.lzma → ...s/typecode/data/extractible/test.tar.lzma
diff --git a/...code/data/contenttype/archive/test.tar.xz → tests/typecode/data/extractible/test.tar.xz b/...code/data/contenttype/archive/test.tar.xz → tests/typecode/data/extractible/test.tar.xz
diff --git a/...ypecode/data/contenttype/archive/test.zip → tests/typecode/data/extractible/test.zip b/...ypecode/data/contenttype/archive/test.zip → tests/typecode/data/extractible/test.zip
diff --git a/.../data/contenttype/archive/win-archive.lib → ...typecode/data/extractible/win-archive.lib b/.../data/contenttype/archive/win-archive.lib → ...typecode/data/extractible/win-archive.lib
diff --git a/tests/typecode/data/filetest/archive/a.tar.gz b/tests/typecode/data/filetest/archive/a.tar.gz
diff --git a/tests/typecode/data/filetest/archive/a.tar.gz.yml b/tests/typecode/data/filetest/archive/a.tar.gz.yml
@@ -0,0 +1,10 @@
+filetype_file: gzip compressed data
+mimetype_file: application/gzip
+mimetype_python: application/x-tar
+is_file: yes
+is_regular: yes
+size: 225
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
diff --git a/...ypecode/data/contenttype/binary/dbase.fdt → .../typecode/data/filetest/archive/dbase.fdt b/...ypecode/data/contenttype/binary/dbase.fdt → .../typecode/data/filetest/archive/dbase.fdt
diff --git a/tests/typecode/data/filetest/archive/dbase.fdt.yml b/tests/typecode/data/filetest/archive/dbase.fdt.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 183
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/e.tar b/tests/typecode/data/filetest/archive/e.tar
diff --git a/tests/typecode/data/filetest/archive/e.tar.bz2 b/tests/typecode/data/filetest/archive/e.tar.bz2
diff --git a/tests/typecode/data/filetest/archive/e.tar.bz2.yml b/tests/typecode/data/filetest/archive/e.tar.bz2.yml
@@ -0,0 +1,9 @@
+filetype_file: bzip2 compressed data, block size = 900k
+mimetype_file: application/x-bzip2
+is_file: yes
+is_regular: yes
+size: 216
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/e.tar.gz b/tests/typecode/data/filetest/archive/e.tar.gz
diff --git a/tests/typecode/data/filetest/archive/e.tar.gz.yml b/tests/typecode/data/filetest/archive/e.tar.gz.yml
@@ -0,0 +1,9 @@
+filetype_file: gzip compressed data
+mimetype_file: application/gzip
+is_file: yes
+is_regular: yes
+size: 217
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/e.tar.yml b/tests/typecode/data/filetest/archive/e.tar.yml
@@ -0,0 +1,10 @@
+filetype_file: POSIX tar archive (GNU)
+mimetype_file: application/x-tar
+is_file: yes
+is_regular: yes
+size: 10240
+is_binary: yes
+is_data: yes
+is_archive: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/file_4.26-1.diff.gz b/tests/typecode/data/filetest/archive/file_4.26-1.diff.gz
diff --git a/tests/typecode/data/filetest/archive/file_4.26-1.diff.gz.yml b/tests/typecode/data/filetest/archive/file_4.26-1.diff.gz.yml
@@ -0,0 +1,9 @@
+filetype_file: gzip compressed data
+mimetype_file: application/gzip
+is_file: yes
+is_regular: yes
+size: 42754
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/posixnotgnu.tar b/tests/typecode/data/filetest/archive/posixnotgnu.tar
diff --git a/tests/typecode/data/filetest/archive/posixnotgnu.tar.yml b/tests/typecode/data/filetest/archive/posixnotgnu.tar.yml
@@ -0,0 +1,9 @@
+filetype_file: POSIX tar archive
+mimetype_file: application/x-tar
+is_file: yes
+is_regular: yes
+size: 35328
+is_binary: yes
+is_archive: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/test.tar.lzma b/tests/typecode/data/filetest/archive/test.tar.lzma
diff --git a/tests/typecode/data/filetest/archive/test.tar.lzma.yml b/tests/typecode/data/filetest/archive/test.tar.lzma.yml
@@ -0,0 +1,9 @@
+filetype_file: LZMA compressed data, streamed
+mimetype_file: application/x-lzma
+is_file: yes
+is_regular: yes
+size: '198'
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/test.tar.xz b/tests/typecode/data/filetest/archive/test.tar.xz
diff --git a/tests/typecode/data/filetest/archive/test.tar.xz.yml b/tests/typecode/data/filetest/archive/test.tar.xz.yml
@@ -0,0 +1,9 @@
+filetype_file: XZ compressed data
+mimetype_file: application/x-xz
+is_file: yes
+is_regular: yes
+size: 2916
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/archive/test.zip b/tests/typecode/data/filetest/archive/test.zip
diff --git a/tests/typecode/data/filetest/archive/test.zip.yml b/tests/typecode/data/filetest/archive/test.zip.yml
@@ -0,0 +1,10 @@
+filetype_file: Zip archive data, at least v1.0 to extract
+mimetype_file: application/zip
+is_file: yes
+is_regular: yes
+size: 177
+is_binary: yes
+is_archive: yes
+is_compressed: yes
+is_media_with_meta: yes
+is_package: yes
diff --git a/tests/typecode/data/filetest/archive/win-archive.lib b/tests/typecode/data/filetest/archive/win-archive.lib
diff --git a/tests/typecode/data/filetest/archive/win-archive.lib.yml b/tests/typecode/data/filetest/archive/win-archive.lib.yml
@@ -0,0 +1,9 @@
+filetype_file: current ar archive
+mimetype_file: application/x-archive
+is_file: yes
+is_regular: yes
+size: 8845
+is_binary: yes
+is_archive: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_0 → ...ta/filetest/binary-random/binary_random_0 b/...contenttype/binary-random/binary_random_0 → ...ta/filetest/binary-random/binary_random_0
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_0.yml b/tests/typecode/data/filetest/binary-random/binary_random_0.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_1 → ...ta/filetest/binary-random/binary_random_1 b/...contenttype/binary-random/binary_random_1 → ...ta/filetest/binary-random/binary_random_1
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_1.yml b/tests/typecode/data/filetest/binary-random/binary_random_1.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_2 → ...ta/filetest/binary-random/binary_random_2 b/...contenttype/binary-random/binary_random_2 → ...ta/filetest/binary-random/binary_random_2
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_2.yml b/tests/typecode/data/filetest/binary-random/binary_random_2.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_3 → ...ta/filetest/binary-random/binary_random_3 b/...contenttype/binary-random/binary_random_3 → ...ta/filetest/binary-random/binary_random_3
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_3.yml b/tests/typecode/data/filetest/binary-random/binary_random_3.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_4 → ...ta/filetest/binary-random/binary_random_4 b/...contenttype/binary-random/binary_random_4 → ...ta/filetest/binary-random/binary_random_4
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_4.yml b/tests/typecode/data/filetest/binary-random/binary_random_4.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_5 → ...ta/filetest/binary-random/binary_random_5 b/...contenttype/binary-random/binary_random_5 → ...ta/filetest/binary-random/binary_random_5
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_5.yml b/tests/typecode/data/filetest/binary-random/binary_random_5.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_6 → ...ta/filetest/binary-random/binary_random_6 b/...contenttype/binary-random/binary_random_6 → ...ta/filetest/binary-random/binary_random_6
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_6.yml b/tests/typecode/data/filetest/binary-random/binary_random_6.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_7 → ...ta/filetest/binary-random/binary_random_7 b/...contenttype/binary-random/binary_random_7 → ...ta/filetest/binary-random/binary_random_7
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_7.yml b/tests/typecode/data/filetest/binary-random/binary_random_7.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...contenttype/binary-random/binary_random_8 → ...ta/filetest/binary-random/binary_random_8 b/...contenttype/binary-random/binary_random_8 → ...ta/filetest/binary-random/binary_random_8
diff --git a/tests/typecode/data/filetest/binary-random/binary_random_8.yml b/tests/typecode/data/filetest/binary-random/binary_random_8.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 1000
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...typecode/data/contenttype/binary/data.fdt → tests/typecode/data/filetest/binary/data.fdt b/...typecode/data/contenttype/binary/data.fdt → tests/typecode/data/filetest/binary/data.fdt
diff --git a/tests/typecode/data/filetest/binary/data.fdt.yml b/tests/typecode/data/filetest/binary/data.fdt.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 183
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/tests/typecode/data/filetest/binary/dbase.fdt b/tests/typecode/data/filetest/binary/dbase.fdt
diff --git a/tests/typecode/data/filetest/binary/dbase.fdt.yml b/tests/typecode/data/filetest/binary/dbase.fdt.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 183
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...type/binary/jruby_time_zone_TimeOfDay.dat → ...test/binary/jruby_time_zone_TimeOfDay.dat b/...type/binary/jruby_time_zone_TimeOfDay.dat → ...test/binary/jruby_time_zone_TimeOfDay.dat
diff --git a/tests/typecode/data/filetest/binary/jruby_time_zone_TimeOfDay.dat.yml b/tests/typecode/data/filetest/binary/jruby_time_zone_TimeOfDay.dat.yml
@@ -0,0 +1,8 @@
+filetype_file: Java serialization data, version 5
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 299
+is_binary: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...e/data/contenttype/binary/pixelstream.rgb → ...code/data/filetest/binary/pixelstream.rgb b/...e/data/contenttype/binary/pixelstream.rgb → ...code/data/filetest/binary/pixelstream.rgb
diff --git a/tests/typecode/data/filetest/binary/pixelstream.rgb.yml b/tests/typecode/data/filetest/binary/pixelstream.rgb.yml
@@ -0,0 +1,9 @@
+filetype_file: data
+mimetype_file: application/octet-stream
+is_file: yes
+is_regular: yes
+size: 129600
+is_binary: yes
+is_data: yes
+contains_text: yes
+is_media_with_meta: yes
diff --git a/...ecode/data/contenttype/binary/windows.dll → ...typecode/data/filetest/binary/windows.dll b/...ecode/data/contenttype/binary/windows.dll → ...typecode/data/filetest/binary/windows.dll
diff --git a/tests/typecode/data/filetest/binary/windows.dll.yml b/tests/typecode/data/filetest/binary/windows.dll.yml
@@ -0,0 +1,9 @@
+filetype_file: PE32+ executable (DLL) x86-64, for MS Windows
+mimetype_file: application/x-dosexec
+is_file: yes
+is_regular: yes
+size: 3394
+is_binary: yes
+contains_text: yes
+is_media_with_meta: yes
+is_winexe: yes
diff --git a/.../typecode/data/contenttype/build/Makefile → tests/typecode/data/filetest/build/Makefile b/.../typecode/data/contenttype/build/Makefile → tests/typecode/data/filetest/build/Makefile
diff --git a/...ecode/data/contenttype/build/Makefile.inc → ...typecode/data/filetest/build/Makefile.inc b/...ecode/data/contenttype/build/Makefile.inc → ...typecode/data/filetest/build/Makefile.inc
diff --git a/tests/typecode/data/filetest/build/Makefile.inc.yml b/tests/typecode/data/filetest/build/Makefile.inc.yml
@@ -0,0 +1,8 @@
+filetype_file: makefile script, ASCII text, with CRLF line terminators
+mimetype_file: text/x-makefile
+is_file: yes
+is_regular: yes
+size: 2888
+is_text: yes
+contains_text: yes
+is_media_with_meta: yes