Implement new library loading approach

We now load the libmagic native library and its database from paths found in:
1. environment variables,
2. OR a location provider plugin,
3. OR the system PATH,
4. OR we fail with an informative error message.

Based on original code contributed by @priv-kweihmann to
scancode-plugins in aboutcode-org/scancode-plugins#9 and
moved here and adapted for use in the core code rather than in a plugin.

Contributed-by: Konrad Weihmann <kweihmann@outlook.com>
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/typecode/magic2.py

@@ -45,17 +45,33 @@
 import os
 
 from commoncode import command
-from plugincode.location_provider import get_location
-
-from os import fsencode
-
-TRACE = False
+from commoncode.system import on_windows
 
 """
 magic2 is minimal and specialized wrapper around a vendored libmagic file
 identification library. This is NOT thread-safe. It is based on python-magic
 by Adam Hup and adapted to the specific needs of ScanCode.
 """
+
+# Tracing flag
+TRACE = True
+
+
+def logger_debug(*args):
+    pass
+
+
+if TRACE:
+    import logging
+    import sys
+
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
+
 #
 # Cached detectors
 #
@@ -74,24 +90,97 @@
 DETECT_ENC = MAGIC_NONE | MAGIC_MIME | MAGIC_MIME_ENCODING
 
 # keys for plugin-provided locations
-TYPECODE_LIBMAGIC_LIBDIR = 'typecode.libmagic.libdir'
 TYPECODE_LIBMAGIC_DLL = 'typecode.libmagic.dll'
-TYPECODE_LIBMAGIC_DATABASE = 'typecode.libmagic.db'
+TYPECODE_LIBMAGIC_DB = 'typecode.libmagic.db'
+
+TYPECODE_LIBMAGIC_PATH_ENVVAR = 'TYPECODE_LIBMAGIC_PATH'
+TYPECODE_LIBMAGIC_DB_PATH_ENVVAR = 'TYPECODE_LIBMAGIC_DB_PATH'
 
 
 def load_lib():
     """
-    Return the loaded libmagic shared library object from plugin-provided path.
+    Return the libmagic shared library object loaded from either:
+    - an environment variable ``TYPECODE_LIBMAGIC_PATH``
+    - a plugin-provided path,
+    - the system PATH.
+    Raise an Exception if no libmagic can be found.
     """
-    dll = get_location(TYPECODE_LIBMAGIC_DLL)
-    libdir = get_location(TYPECODE_LIBMAGIC_LIBDIR)
-    if not (dll and libdir) or not os.path.isfile(dll) or not os.path.isdir(libdir):
+    from plugincode.location_provider import get_location
+
+    # try the environment first
+    dll_loc = os.environ.get(TYPECODE_LIBMAGIC_PATH_ENVVAR)
+
+    if TRACE and dll_loc:
+        logger_debug('load_lib:', 'got environ magic location:', dll_loc)
+
+    # try a plugin-provided path second
+    if not dll_loc:
+        dll_loc = get_location(TYPECODE_LIBMAGIC_DLL)
+
+        if TRACE and dll_loc:
+            logger_debug('load_lib:', 'got plugin magic location:', dll_loc)
+
+    # try the PATH
+    if not dll_loc:
+        dll = 'libmagic.dll' if on_windows else 'libmagic.so'
+        dll_loc = command.find_in_path(dll)
+
+        if TRACE and dll_loc:
+            logger_debug('load_lib:', 'got path magic location:', dll_loc)
+
+    if not dll_loc or not os.path.isfile(dll_loc):
+        raise Exception(
+            'CRITICAL: libmagic DLL and its magic database are not installed. '
+            'Unable to continue: you need to install a valid typecode-libmagic '
+            'plugin with a valid and proper libmagic and magic DB available. '
+            f'OR set the {TYPECODE_LIBMAGIC_PATH_ENVVAR} environment variable.'
+    )
+    return command.load_shared_library(dll_loc)
+
+
+def get_magicdb_location(_cache=[]):
+    """
+    Return the location of the magicdb loaded from either:
+    - an environment variable ``TYPECODE_LIBMAGIC_DB_PATH``,
+    - a plugin-provided path,
+    - the system PATH.
+    Raise an Exception if no magicdb command can be found.
+    """
+    if _cache:
+        return _cache[0]
+
+    from plugincode.location_provider import get_location
+
+    # try the environment first
+    magicdb_loc = os.environ.get(TYPECODE_LIBMAGIC_DB_PATH_ENVVAR)
+
+    if TRACE and magicdb_loc:
+        logger_debug('get_magicdb_location:', 'got environ magicdb location:', magicdb_loc)
+
+    # try a plugin-provided path second
+    if not magicdb_loc:
+        magicdb_loc = get_location(TYPECODE_LIBMAGIC_DB)
+
+        if TRACE and magicdb_loc:
+            logger_debug('get_magicdb_location:', 'got plugin magicdb location:', magicdb_loc)
+
+    # try the PATH
+    if not magicdb_loc:
+        db = 'magic.mgc'
+        magicdb_loc = command.find_in_path(db)
+
+        if TRACE and magicdb_loc:
+            logger_debug('get_magicdb_location:', 'got path magicdb location:', magicdb_loc)
+
+    if not magicdb_loc or not os.path.isfile(magicdb_loc):
         raise Exception(
-            'CRITICAL: libmagic DLL and is magic database are not installed. '
+            'CRITICAL: Libmagic magic database is not installed. '
             'Unable to continue: you need to install a valid typecode-libmagic '
-            'plugin with a valid and proper libmagic and magic DB available.'
+            'plugin with a valid magic database available. '
+            'OR set the TYPECODE_LIBMAGIC_DB_PATH environment variable.'
     )
-    return command.load_shared_library(dll, libdir)
+    _cache.append(magicdb_loc)
+    return magicdb_loc
 
 
 if TRACE:
@@ -164,11 +253,11 @@ def __init__(self, flags, magic_db_location=None):
         self.flags = flags
         self.cookie = _magic_open(self.flags)
         if not magic_db_location:
-            magic_db_location = get_location(TYPECODE_LIBMAGIC_DATABASE)
+            magic_db_location = get_magicdb_location()
 
-        # Note: this location must always be bytes on Python2 and 3, all OSes
+        # Note: this location must always be FS-encoded bytes on all OSes
         if isinstance(magic_db_location, str):
-            magic_db_location = fsencode(magic_db_location)
+            magic_db_location = os.fsencode(magic_db_location)
 
         _magic_load(self.cookie, magic_db_location)
 
@@ -190,7 +279,7 @@ def get(self, location):
             # location string may therefore be mangled and the file not accessible
             # anymore by libmagic in some cases.
             try:
-                uloc = fsencode(location)
+                uloc = os.fsencode(location)
                 return  _magic_file(self.cookie, uloc)
             except:
                 # if all fails, read the start of the file instead