Merge pull request #758 from nexB/688-fsencode-fsdecode-755-unicode-test-files

Ensure fsencode/fsdecode works on ASCII FS.  Use tarball for unicode test files

Author: pombredanne

setup.py

@@ -143,7 +143,7 @@ def read(*names, **kwargs):
         'bz2file >= 0.98',
 
         # commoncode
-        'backports.os == 0.1rc1',
+        'backports.os == 0.1.1',
         'future >= 0.16.0, < 0.17.0',
         'text-unidecode >= 1.0, < 2.0',
 

src/commoncode/command.py

@@ -53,6 +53,11 @@
     # Python 3
     unicode = str
 
+try:
+    from os import fsencode
+except ImportError:
+    from backports.os import fsencode
+
 
 """
 Minimal wrapper for executing external commands in sub-processes. The approach
@@ -344,7 +349,7 @@ def load_lib(libname, root_dir):
     if os.path.exists(so):
         if not isinstance(so, bytes):
             # ensure that the path is not Unicode...
-            so = so.encode(fileutils.FS_ENCODING)
+            so = fsencode(so)
         lib = ctypes.CDLL(so)
         if lib and lib._name:
             return lib

src/commoncode/fileutils.py

@@ -35,6 +35,13 @@
     # Python 3
     unicode = str
 
+try:
+    from os import fsencode
+except ImportError:
+    from backports.os import fsencode
+    from backports.os import fsdecode
+
+
 import codecs
 import errno
 import os
@@ -45,7 +52,6 @@
 import sys
 import tempfile
 
-from backports import os as osb
 
 from commoncode import filetype
 from commoncode.filetype import is_rwx
@@ -77,8 +83,6 @@ def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args))
 
 
-FS_ENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
-
 # Paths can only be sanely handled as raw bytes on Linux
 PATH_TYPE = bytes if on_linux else unicode
 POSIX_PATH_SEP = b'/' if on_linux else '/'
@@ -217,8 +221,8 @@ def path_to_unicode(path):
     """
     if isinstance(path, unicode):
         return path
-    if TRACE: logger_debug('path_to_unicode:', osb.fsdecode(path))
-    return osb.fsdecode(path)
+    if TRACE: logger_debug('path_to_unicode:', fsdecode(path))
+    return fsdecode(path)
 
 
 def path_to_bytes(path):
@@ -227,8 +231,8 @@ def path_to_bytes(path):
     """
     if isinstance(path, bytes):
         return path
-    if TRACE: logger_debug('path_to_bytes:' , repr(osb.fsencode(path)))
-    return osb.fsencode(path)
+    if TRACE: logger_debug('path_to_bytes:' , repr(fsencode(path)))
+    return fsencode(path)
 
 
 def is_posixpath(location):

src/commoncode/testcase.py

@@ -28,14 +28,14 @@
 from __future__ import division
 from __future__ import unicode_literals
 
-from unittest import TestCase as TestCaseClass
-
 import filecmp
+from functools import partial
 import os
 import shutil
 import stat
 import sys
 import tarfile
+from unittest import TestCase as TestCaseClass
 import zipfile
 
 from commoncode import fileutils
@@ -278,19 +278,27 @@ def extract_test_tar(self, test_path, verbatim=False):
     def extract_test_tar_raw(self, test_path, *args, **kwargs):
         return self.__extract(test_path, extract_tar_raw)
 
+    def extract_test_tar_unicode(self, test_path, *args, **kwargs):
+        return self.__extract(test_path, extract_tar_uni)
 
-def extract_tar_raw(test_path, target_dir, *args, **kwargs):
+
+def _extract_tar_raw(test_path, target_dir, to_bytes, *args, **kwargs):
     """
     Raw simplified extract for certain really weird paths and file
     names.
     """
-    # use bytes for paths on ALL OSes (though this may fail on macOS)
-    target_dir = path_to_bytes(target_dir)
-    test_path = path_to_bytes(test_path)
+    if to_bytes:
+        # use bytes for paths on ALL OSes (though this may fail on macOS)
+        target_dir = path_to_bytes(target_dir)
+        test_path = path_to_bytes(test_path)
     tar = tarfile.open(test_path)
     tar.extractall(path=target_dir)
     tar.close()
 
+extract_tar_raw = partial(_extract_tar_raw, to_bytes=True)
+
+extract_tar_uni = partial(_extract_tar_raw, to_bytes=False)
+
 
 def extract_tar(location, target_dir, verbatim=False, *args, **kwargs):
     """

src/extractcode/libarchive2.py

@@ -51,6 +51,13 @@
 from extractcode import ExtractErrorPasswordProtected
 
 
+# Python 2 and 3 support
+try:
+    from os import fsencode
+except ImportError:
+    from backports.os import fsencode
+
+
 logger = logging.getLogger(__name__)
 DEBUG = False
 # logging.basicConfig(level=logging.DEBUG)
@@ -104,7 +111,7 @@ def load_lib():
     if os.path.exists(libarchive):
         if not isinstance(libarchive, bytes):
             # ensure that the path is not Unicode...
-            libarchive = libarchive.encode(fileutils.FS_ENCODING)
+            libarchive = fsencode(libarchive)
         lib = ctypes.CDLL(libarchive)
         if lib and lib._name:
             return lib

src/typecode/magic2.py

@@ -48,11 +48,17 @@
 
 import os.path
 import ctypes
-import sys
 
 from commoncode import system
 from commoncode import command
 
+# Python 2 and 3 support
+try:
+    from os import fsencode
+except ImportError:
+    from backports.os import fsencode
+
+
 """
 magic2 is minimal and specialized wrapper around a vendored libmagic file
 identification library. This is NOT thread-safe. It is based on python-magic
@@ -204,7 +210,7 @@ def load_lib():
     if os.path.exists(magic_so):
         if not isinstance(magic_so, bytes):
             # ensure that the path is not Unicode...
-            magic_so = magic_so.encode(sys.getfilesystemencoding() or sys.getdefaultencoding())
+            magic_so = fsencode(magic_so)
         lib = ctypes.CDLL(magic_so)
         if lib and lib._name:
             return lib