Allow symlinks in is_file, is_dir, and walk

    * Add missing dep `typing`

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

setup.cfg

@@ -6,7 +6,7 @@ license_file = apache-2.0.LICENSE
 name = commoncode
 author = nexB. Inc. and others
 author_email = info@aboutcode.org
-description = commoncode 
+description = commoncode
 long_description = file:README.rst
 url = https://github.com/nexB/commoncode
 classifiers =
@@ -16,7 +16,7 @@ classifiers =
     Programming Language :: Python :: 3
     Topic :: Software Development
     Topic :: Utilities
-keywords = 
+keywords =
 
 [options]
 package_dir=
@@ -33,6 +33,7 @@ install_requires =
     requests >= 2.7.0, < 3.0.0
     intbitset >= 2.3.0, < 3.0
     saneyaml
+    typing >=3.6, < 3.7
 setup_requires = setuptools_scm >= 4
 
 [options.packages.find]

src/commoncode/filetype.py

@@ -45,20 +45,24 @@ def is_link(location):
     return location and os.path.islink(location)
 
 
-def is_file(location):
+def is_file(location, allow_symlinks=False):
     """
     Return True if `location` is a file.
     """
-    return (location and os.path.isfile(location)
-            and not is_link(location) and not is_broken_link(location))
+    _is_file = location and os.path.isfile(location)
+    if allow_symlinks:
+        return _is_file
+    return _is_file and not is_link(location) and not is_broken_link(location)
 
 
-def is_dir(location):
+def is_dir(location, allow_symlinks=False):
     """
     Return True if `location` is a directory.
     """
-    return (location and os.path.isdir(location) and not is_file(location)
-            and not is_link(location) and not is_broken_link(location))
+    _is_dir = location and os.path.isdir(location) and not is_file(location)
+    if allow_symlinks:
+        return _is_dir
+    return _is_dir and not is_link(location) and not is_broken_link(location)
 
 
 def is_regular(location):

src/commoncode/fileutils.py

@@ -197,7 +197,7 @@ def prepare_path(pth):
         if not isinstance(pth, bytes):
             pth = fsencode(pth)
         return pth
-    else:    
+    else:
         if not isinstance(pth, compat.unicode):
             return fsdecode(pth)
     return pth
@@ -365,12 +365,12 @@ def ignore_nothing(_):
     return False
 
 
-def walk(location, ignored=None):
+def walk(location, ignored=None, allow_symlinks=False):
     """
     Walk location returning the same tuples as os.walk but with a different
     behavior:
      - always walk top-down, breadth-first.
-     - always ignore and never follow symlinks,
+     - always ignore and never follow symlinks (unless `allow_symlinks` is True),
      - always ignore special files (FIFOs, etc.)
      - optionally ignore files and directories by invoking the `ignored`
        callable on files and directories returning True if it should be ignored.
@@ -387,33 +387,38 @@ def walk(location, ignored=None):
             logger_debug('walk: ignored:', location, is_ignored)
         return
 
-    if filetype.is_file(location) :
+    if filetype.is_file(location, allow_symlinks=allow_symlinks) :
         yield parent_directory(location), [], [file_name(location)]
 
-    elif filetype.is_dir(location):
+    elif filetype.is_dir(location, allow_symlinks=allow_symlinks):
         dirs = []
         files = []
         # TODO: consider using scandir
         for name in os.listdir(location):
             loc = os.path.join(location, name)
             if filetype.is_special(loc) or (ignored and ignored(loc)):
-                if TRACE:
-                    ign = ignored and ignored(loc)
-                    logger_debug('walk: ignored:', loc, ign)
-                continue
+                if (allow_symlinks
+                        and filetype.is_link(loc)
+                        and not filetype.is_broken_link(location)):
+                    pass
+                else:
+                    if TRACE:
+                        ign = ignored and ignored(loc)
+                        logger_debug('walk: ignored:', loc, ign)
+                    continue
             # special files and symlinks are always ignored
-            if filetype.is_dir(loc):
+            if filetype.is_dir(loc, allow_symlinks=allow_symlinks):
                 dirs.append(name)
-            elif filetype.is_file(loc):
+            elif filetype.is_file(loc, allow_symlinks=allow_symlinks):
                 files.append(name)
         yield location, dirs, files
 
         for dr in dirs:
-            for tripple in walk(os.path.join(location, dr), ignored):
+            for tripple in walk(os.path.join(location, dr), ignored, allow_symlinks=allow_symlinks):
                 yield tripple
 
 
-def resource_iter(location, ignored=ignore_nothing, with_dirs=True):
+def resource_iter(location, ignored=ignore_nothing, with_dirs=True, allow_symlinks=False):
     """
     Return an iterable of paths at `location` recursively.
 
@@ -424,7 +429,7 @@ def resource_iter(location, ignored=ignore_nothing, with_dirs=True):
     """
     if on_linux and py2:
         location = fsencode(location)
-    for top, dirs, files in walk(location, ignored):
+    for top, dirs, files in walk(location, ignored, allow_symlinks=allow_symlinks):
         if with_dirs:
             for d in dirs:
                 yield os.path.join(top, d)