From dba9caeb29db0349adfc0281717408315dac9b8a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 14 Mar 2018 12:24:43 -0700 Subject: [PATCH 01/10] Deterministic pipfile ordering (#22) * Ignore local temp directory Signed-off-by: Philippe Ombredanne * Use ordered mapping when loding Pipfile/Pipfile.lock * this preserves ordering and fix #19 * catch ValueError rather than json.JSONDecodeError which is a simplesjon exception and not a json module error Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + dparse/parser.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 7beff4e..d15134c 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ target/ # pyenv python configuration file .python-version sandbox.py +/tmp/ diff --git a/dparse/parser.py b/dparse/parser.py index 9823991..01329bd 100644 --- a/dparse/parser.py +++ b/dparse/parser.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, absolute_import +from collections import OrderedDict import re import yaml @@ -338,7 +339,7 @@ def parse(self): :return: """ try: - data = toml.loads(self.obj.content) + data = toml.loads(self.obj.content, _dict=OrderedDict) if data: for package_type in ['packages', 'dev-packages']: if package_type in data: @@ -368,7 +369,7 @@ def parse(self): :return: """ try: - data = json.loads(self.obj.content) + data = json.loads(self.obj.content, object_pairs_hook=OrderedDict) if data: for package_type in ['default', 'develop']: if package_type in data: @@ -387,7 +388,7 @@ def parse(self): section=package_type ) ) - except json.JSONDecodeError: + except ValueError: pass From be377852aceeec0fbedd7461afe2c639ffab8ffa Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Thu, 15 Mar 2018 12:28:31 +0100 Subject: [PATCH 02/10] new release, 0.3.0 --- HISTORY.rst | 8 ++++++++ dparse/__init__.py | 4 ++-- setup.py | 3 +-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 054b4b2..c9d432d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,14 @@ History ======= +0.3.0 (2018-03-01) +------------------ + +* Added support for setup.cfg files (thanks @kexepal) +* Dependencies from Pipfiles now include the section (thanks @paulortman) +* Multiline requirements are now ignored if they are marked +* Added experimental support for Pipfiles + 0.2.1 (2017-07-19) ------------------ diff --git a/dparse/__init__.py b/dparse/__init__.py index 6073b0e..824cca1 100644 --- a/dparse/__init__.py +++ b/dparse/__init__.py @@ -3,7 +3,7 @@ """Top-level package for Dependency Parser.""" __author__ = """Jannis Gebauer""" -__email__ = 'ja.geb@me.com' -__version__ = '0.2.1' +__email__ = 'support@pyup.io' +__version__ = '0.3.0' from .parser import parse diff --git a/setup.py b/setup.py index 98ff0b6..fc119b8 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name='dparse', - version='0.2.1', + version='0.3.0', description="A parser for Python dependency files", long_description=readme + '\n\n' + history, author="Jannis Gebauer", @@ -51,7 +51,6 @@ 'License :: OSI Approved :: MIT License', 'Natural Language :: English', "Programming Language :: Python :: 2", - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', From 5dfc7fc62db11bde56473364a24be38c5ec501b3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 4 Apr 2018 09:30:28 +0200 Subject: [PATCH 03/10] Support invalid toml Pipfile (#23) * add tests and workaround a toml bug * https://github.com/uiri/toml/issues/156 Signed-off-by: Philippe Ombredanne --- dparse/parser.py | 3 +-- tests/test_parse.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dparse/parser.py b/dparse/parser.py index 01329bd..05de850 100644 --- a/dparse/parser.py +++ b/dparse/parser.py @@ -357,10 +357,9 @@ def parse(self): section=package_type ) ) - except toml.TomlDecodeError: + except (toml.TomlDecodeError, IndexError) as e: pass - class PipfileLockParser(Parser): def parse(self): diff --git a/tests/test_parse.py b/tests/test_parse.py index 72f3822..6e1e35b 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -290,3 +290,25 @@ def test_pipfile_lock(): "sha256:52475f607c92035d4ac8fee284f56213065a4a6b25ed43f7e39df0e576e69e9f", "sha256:d96b804be412a5125a594023ec524a2010a6ffa4d408e5482ab6ff3cb97ec12f" ] + + +def test_pipfile_with_invalid_toml(): + content = """[[source] + +url = "http://some.pypi.mirror.server.org/simple" +verify_ssl = false +ds name < "pypi" +""" + dep_file = parse(content, file_type=filetypes.pipfile) + assert not dep_file.dependencies + + +def test_pipfile_lock_with_invalid_json(): + content = """{ + "_meta": + "hash": { + "sha256": "8b5635a4f7b069ae6661115b9eaa15466f7cd96794af5d131735a3638be101fb" + }, +}""" + dep_file = parse(content, file_type=filetypes.pipfile_lock) + assert not dep_file.dependencies From d574d3b03f87e2aba81b0c5ef0fc5a6537cd1bbc Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:00:32 +0200 Subject: [PATCH 04/10] make pipenv optional for updater only --- .pytest_cache/v/cache/lastfailed | 3 + .pytest_cache/v/cache/nodeids | 49 ++ dparse/parser.py | 2 +- dparse/updater.py | 8 +- dparse/vendor/__init__.py | 0 dparse/vendor/toml.py | 923 +++++++++++++++++++++++++++++++ requirements_dev.txt | 1 + setup.py | 4 +- 8 files changed, 986 insertions(+), 4 deletions(-) create mode 100644 .pytest_cache/v/cache/lastfailed create mode 100644 .pytest_cache/v/cache/nodeids create mode 100644 dparse/vendor/__init__.py create mode 100644 dparse/vendor/toml.py diff --git a/.pytest_cache/v/cache/lastfailed b/.pytest_cache/v/cache/lastfailed new file mode 100644 index 0000000..609de55 --- /dev/null +++ b/.pytest_cache/v/cache/lastfailed @@ -0,0 +1,3 @@ +{ + "tests/test_updater.py::test_update_pipfile": true +} \ No newline at end of file diff --git a/.pytest_cache/v/cache/nodeids b/.pytest_cache/v/cache/nodeids new file mode 100644 index 0000000..ba00e17 --- /dev/null +++ b/.pytest_cache/v/cache/nodeids @@ -0,0 +1,49 @@ +[ + "tests/test_dependencies.py::test_dependency_serialize", + "tests/test_dependencies.py::test_dependency_deserialize", + "tests/test_dependencies.py::test_dependency_file_serialize", + "tests/test_dependencies.py::test_dependency_file_deserialize", + "tests/test_dependencies.py::test_parser_class", + "tests/test_parse.py::test_requirements_with_invalid_requirement", + "tests/test_parse.py::test_tox_ini_with_invalid_requirement", + "tests/test_parse.py::test_conda_file_with_invalid_requirement", + "tests/test_parse.py::test_conda_file_invalid_yml", + "tests/test_parse.py::test_conda_file_marked_line", + "tests/test_parse.py::test_tox_ini_marked_line", + "tests/test_parse.py::test_resolve_file", + "tests/test_parse.py::test_index_server", + "tests/test_parse.py::test_requirements_package_with_index_server", + "tests/test_parse.py::test_requirements_parse_empty_line", + "tests/test_parse.py::test_requirements_parse_unsupported_line_start", + "tests/test_parse.py::test_file_resolver", + "tests/test_parse.py::test_is_marked_file", + "tests/test_parse.py::test_is_marked_line", + "tests/test_parse.py::test_pipfile", + "tests/test_parse.py::test_pipfile_lock", + "tests/test_updater.py::test_update_tox_ini", + "tests/test_updater.py::test_update_conda_yml", + "tests/test_updater.py::test_update_requirements_multispace", + "tests/test_updater.py::test_update_requirements_compatible", + "tests/test_updater.py::test_update_requirements_compatible_matching_latest", + "tests/test_updater.py::test_update_requirements_contains_correct_sep_char", + "tests/test_updater.py::test_update_requirements_with_hashes", + "tests/test_updater.py::test_update_requirements_with_hashes_and_comment_and_env_markers", + "tests/test_updater.py::test_update_requirements_with_hashes_and_comment_inline", + "tests/test_updater.py::test_update_requirements_with_hash_and_space_separator", + "tests/test_updater.py::test_update_requirements_with_hash_and_comment_and_env_markers_inline", + "tests/test_updater.py::test_update_requirements_with_hash_inline", + "tests/test_updater.py::test_update_requirements_with_env_markers", + "tests/test_updater.py::test_update_requirements_with_env_markers_and_comment", + "tests/test_updater.py::test_update_requirements_with_extras", + "tests/test_updater.py::test_update_requirements_with_tabs", + "tests/test_updater.py::test_update_requirements_with_plus", + "tests/test_updater.py::test_update_requirements_line_endings", + "tests/test_updater.py::test_update_requirements_simple_pinned", + "tests/test_updater.py::test_update_requirements_simple_unpinned", + "tests/test_updater.py::test_update_requirements_simple_unpinned_with_comment", + "tests/test_updater.py::test_update_requirements_cookiecutter_template", + "tests/test_updater.py::test_update_requirements_with_double_package_name", + "tests/test_updater.py::test_update_requirements_ranged", + "tests/test_updater.py::test_update_requirements_unfinished_line", + "tests/test_updater.py::test_update_pipfile" +] \ No newline at end of file diff --git a/dparse/parser.py b/dparse/parser.py index 01329bd..697860c 100644 --- a/dparse/parser.py +++ b/dparse/parser.py @@ -28,7 +28,7 @@ from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement import six from . import filetypes -from pipenv.vendor import toml +from .vendor import toml from packaging.specifiers import SpecifierSet import json diff --git a/dparse/updater.py b/dparse/updater.py index 03cd6f6..7acbb5a 100644 --- a/dparse/updater.py +++ b/dparse/updater.py @@ -4,7 +4,7 @@ import json # Python 2 & 3 compatible StringIO import tempfile -import toml +from .vendor import toml import os @@ -85,7 +85,11 @@ def update(cls, content, dependency, version, spec="==", hashes=()): data[package_type][dependency.full_name] = "{spec}{version}".format( spec=spec, version=version ) - from pipenv.project import Project + try: + from pipenv.project import Project + except ImportError: + raise ImportError("Updating a Pipfile requires the pipenv extra to be installed. Install it with " + "pip install dparse[pipenv]") pipfile = tempfile.NamedTemporaryFile(delete=False) p = Project(chdir=False) p.write_toml(data=data, path=pipfile.name) diff --git a/dparse/vendor/__init__.py b/dparse/vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dparse/vendor/toml.py b/dparse/vendor/toml.py new file mode 100644 index 0000000..ebbaed6 --- /dev/null +++ b/dparse/vendor/toml.py @@ -0,0 +1,923 @@ +"""Python module which parses and emits TOML. + +Released under the MIT license. +""" +import re +import io +import datetime +from os import linesep + +__version__ = "0.9.2" +__spec__ = "0.4.0" + + +class TomlDecodeError(Exception): + """Base toml Exception / Error.""" + pass + + +class TomlTz(datetime.tzinfo): + def __init__(self, toml_offset): + if toml_offset == "Z": + self._raw_offset = "+00:00" + else: + self._raw_offset = toml_offset + self._sign = -1 if self._raw_offset[0] == '-' else 1 + self._hours = int(self._raw_offset[1:3]) + self._minutes = int(self._raw_offset[4:6]) + + def tzname(self, dt): + return "UTC" + self._raw_offset + + def utcoffset(self, dt): + return self._sign * datetime.timedelta(hours=self._hours, + minutes=self._minutes) + + def dst(self, dt): + return datetime.timedelta(0) + + +class InlineTableDict(object): + """Sentinel subclass of dict for inline tables.""" + + +def _get_empty_inline_table(_dict): + class DynamicInlineTableDict(_dict, InlineTableDict): + """Concrete sentinel subclass for inline tables. + It is a subclass of _dict which is passed in dynamically at load time + It is also a subclass of InlineTableDict + """ + + return DynamicInlineTableDict() + + +try: + _range = xrange +except NameError: + unicode = str + _range = range + basestring = str + unichr = chr + +try: + FNFError = FileNotFoundError +except NameError: + FNFError = IOError + + +def load(f, _dict=dict): + """Parses named file or files as toml and returns a dictionary + + Args: + f: Path to the file to open, array of files to read into single dict + or a file descriptor + _dict: (optional) Specifies the class of the returned toml dictionary + + Returns: + Parsed toml file represented as a dictionary + + Raises: + TypeError -- When f is invalid type + TomlDecodeError: Error while decoding toml + IOError / FileNotFoundError -- When an array with no valid (existing) + (Python 2 / Python 3) file paths is passed + """ + + if isinstance(f, basestring): + with io.open(f, encoding='utf-8') as ffile: + return loads(ffile.read(), _dict) + elif isinstance(f, list): + from os import path as op + from warnings import warn + if not [path for path in f if op.exists(path)]: + error_msg = "Load expects a list to contain filenames only." + error_msg += linesep + error_msg += ("The list needs to contain the path of at least one " + "existing file.") + raise FNFError(error_msg) + d = _dict() + for l in f: + if op.exists(l): + d.update(load(l)) + else: + warn("Non-existent filename in list with at least one valid " + "filename") + return d + else: + try: + return loads(f.read(), _dict) + except AttributeError: + raise TypeError("You can only load a file descriptor, filename or " + "list") + + +_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') + + +def loads(s, _dict=dict): + """Parses string as toml + + Args: + s: String to be parsed + _dict: (optional) Specifies the class of the returned toml dictionary + + Returns: + Parsed toml file represented as a dictionary + + Raises: + TypeError: When a non-string is passed + TomlDecodeError: Error while decoding toml + """ + + implicitgroups = [] + retval = _dict() + currentlevel = retval + if not isinstance(s, basestring): + raise TypeError("Expecting something like a string") + + if not isinstance(s, unicode): + s = s.decode('utf8') + + sl = list(s) + openarr = 0 + openstring = False + openstrchar = "" + multilinestr = False + arrayoftables = False + beginline = True + keygroup = False + keyname = 0 + for i, item in enumerate(sl): + if item == '\r' and sl[i + 1] == '\n': + sl[i] = ' ' + continue + if keyname: + if item == '\n': + raise TomlDecodeError("Key name found without value." + " Reached end of line.") + if openstring: + if item == openstrchar: + keyname = 2 + openstring = False + openstrchar = "" + continue + elif keyname == 1: + if item.isspace(): + keyname = 2 + continue + elif item.isalnum() or item == '_' or item == '-': + continue + elif keyname == 2 and item.isspace(): + continue + if item == '=': + keyname = 0 + else: + raise TomlDecodeError("Found invalid character in key name: '" + + item + "'. Try quoting the key name.") + if item == "'" and openstrchar != '"': + k = 1 + try: + while sl[i - k] == "'": + k += 1 + if k == 3: + break + except IndexError: + pass + if k == 3: + multilinestr = not multilinestr + openstring = multilinestr + else: + openstring = not openstring + if openstring: + openstrchar = "'" + else: + openstrchar = "" + if item == '"' and openstrchar != "'": + oddbackslash = False + k = 1 + tripquote = False + try: + while sl[i - k] == '"': + k += 1 + if k == 3: + tripquote = True + break + if k == 1 or (k == 3 and tripquote): + while sl[i - k] == '\\': + oddbackslash = not oddbackslash + k += 1 + except IndexError: + pass + if not oddbackslash: + if tripquote: + multilinestr = not multilinestr + openstring = multilinestr + else: + openstring = not openstring + if openstring: + openstrchar = '"' + else: + openstrchar = "" + if item == '#' and (not openstring and not keygroup and + not arrayoftables): + j = i + try: + while sl[j] != '\n': + sl[j] = ' ' + j += 1 + except IndexError: + break + if item == '[' and (not openstring and not keygroup and + not arrayoftables): + if beginline: + if sl[i + 1] == '[': + arrayoftables = True + else: + keygroup = True + else: + openarr += 1 + if item == ']' and not openstring: + if keygroup: + keygroup = False + elif arrayoftables: + if sl[i - 1] == ']': + arrayoftables = False + else: + openarr -= 1 + if item == '\n': + if openstring or multilinestr: + if not multilinestr: + raise TomlDecodeError("Unbalanced quotes") + if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( + sl[i - 2] == sl[i - 1])): + sl[i] = sl[i - 1] + if sl[i - 3] == sl[i - 1]: + sl[i - 3] = ' ' + elif openarr: + sl[i] = ' ' + else: + beginline = True + elif beginline and sl[i] != ' ' and sl[i] != '\t': + beginline = False + if not keygroup and not arrayoftables: + if sl[i] == '=': + raise TomlDecodeError("Found empty keyname. ") + keyname = 1 + s = ''.join(sl) + s = s.split('\n') + multikey = None + multilinestr = "" + multibackslash = False + for line in s: + if not multilinestr or multibackslash or '\n' not in multilinestr: + line = line.strip() + if line == "" and (not multikey or multibackslash): + continue + if multikey: + if multibackslash: + multilinestr += line + else: + multilinestr += line + multibackslash = False + if len(line) > 2 and (line[-1] == multilinestr[0] and + line[-2] == multilinestr[0] and + line[-3] == multilinestr[0]): + value, vtype = _load_value(multilinestr, _dict) + currentlevel[multikey] = value + multikey = None + multilinestr = "" + else: + k = len(multilinestr) - 1 + while k > -1 and multilinestr[k] == '\\': + multibackslash = not multibackslash + k -= 1 + if multibackslash: + multilinestr = multilinestr[:-1] + else: + multilinestr += "\n" + continue + if line[0] == '[': + arrayoftables = False + if line[1] == '[': + arrayoftables = True + line = line[2:].split(']]', 1) + else: + line = line[1:].split(']', 1) + if line[1].strip() != "": + raise TomlDecodeError("Key group not on a line by itself.") + groups = line[0].split('.') + i = 0 + while i < len(groups): + groups[i] = groups[i].strip() + if groups[i][0] == '"' or groups[i][0] == "'": + groupstr = groups[i] + j = i + 1 + while not groupstr[0] == groupstr[-1]: + j += 1 + groupstr = '.'.join(groups[i:j]) + groups[i] = groupstr[1:-1] + groups[i + 1:j] = [] + else: + if not _groupname_re.match(groups[i]): + raise TomlDecodeError("Invalid group name '" + + groups[i] + "'. Try quoting it.") + i += 1 + currentlevel = retval + for i in _range(len(groups)): + group = groups[i] + if group == "": + raise TomlDecodeError("Can't have a keygroup with an empty " + "name") + try: + currentlevel[group] + if i == len(groups) - 1: + if group in implicitgroups: + implicitgroups.remove(group) + if arrayoftables: + raise TomlDecodeError("An implicitly defined " + "table can't be an array") + elif arrayoftables: + currentlevel[group].append(_dict()) + else: + raise TomlDecodeError("What? " + group + + " already exists?" + + str(currentlevel)) + except TypeError: + currentlevel = currentlevel[-1] + try: + currentlevel[group] + except KeyError: + currentlevel[group] = _dict() + if i == len(groups) - 1 and arrayoftables: + currentlevel[group] = [_dict()] + except KeyError: + if i != len(groups) - 1: + implicitgroups.append(group) + currentlevel[group] = _dict() + if i == len(groups) - 1 and arrayoftables: + currentlevel[group] = [_dict()] + currentlevel = currentlevel[group] + if arrayoftables: + try: + currentlevel = currentlevel[-1] + except KeyError: + pass + elif line[0] == "{": + if line[-1] != "}": + raise TomlDecodeError("Line breaks are not allowed in inline" + "objects") + _load_inline_object(line, currentlevel, _dict, multikey, + multibackslash) + elif "=" in line: + ret = _load_line(line, currentlevel, _dict, multikey, + multibackslash) + if ret is not None: + multikey, multilinestr, multibackslash = ret + return retval + + +def _load_inline_object(line, currentlevel, _dict, multikey=False, + multibackslash=False): + candidate_groups = line[1:-1].split(",") + groups = [] + if len(candidate_groups) == 1 and not candidate_groups[0].strip(): + candidate_groups.pop() + while len(candidate_groups) > 0: + candidate_group = candidate_groups.pop(0) + try: + _, value = candidate_group.split('=', 1) + except ValueError: + raise TomlDecodeError("Invalid inline table encountered") + value = value.strip() + if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( + value[0] in '-0123456789' or + value in ('true', 'false') or + (value[0] == "[" and value[-1] == "]"))): + groups.append(candidate_group) + else: + candidate_groups[0] = candidate_group + "," + candidate_groups[0] + for group in groups: + status = _load_line(group, currentlevel, _dict, multikey, + multibackslash) + if status is not None: + break + + +# Matches a TOML number, which allows underscores for readability +_number_with_underscores = re.compile('([0-9])(_([0-9]))*') + + +def _strictly_valid_num(n): + n = n.strip() + if not n: + return False + if n[0] == '_': + return False + if n[-1] == '_': + return False + if "_." in n or "._" in n: + return False + if len(n) == 1: + return True + if n[0] == '0' and n[1] != '.': + return False + if n[0] == '+' or n[0] == '-': + n = n[1:] + if n[0] == '0' and n[1] != '.': + return False + if '__' in n: + return False + return True + + +def _load_line(line, currentlevel, _dict, multikey, multibackslash): + i = 1 + pair = line.split('=', i) + strictly_valid = _strictly_valid_num(pair[-1]) + if _number_with_underscores.match(pair[-1]): + pair[-1] = pair[-1].replace('_', '') + while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and + pair[-1][0] != "'" and pair[-1][0] != '"' and + pair[-1][0] != '[' and pair[-1][0] != '{' and + pair[-1] != 'true' and pair[-1] != 'false'): + try: + float(pair[-1]) + break + except ValueError: + pass + if _load_date(pair[-1]) is not None: + break + i += 1 + prev_val = pair[-1] + pair = line.split('=', i) + if prev_val == pair[-1]: + raise TomlDecodeError("Invalid date or number") + if strictly_valid: + strictly_valid = _strictly_valid_num(pair[-1]) + pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] + if (pair[0][0] == '"' or pair[0][0] == "'") and \ + (pair[0][-1] == '"' or pair[0][-1] == "'"): + pair[0] = pair[0][1:-1] + if len(pair[1]) > 2 and ((pair[1][0] == '"' or pair[1][0] == "'") and + pair[1][1] == pair[1][0] and + pair[1][2] == pair[1][0] and + not (len(pair[1]) > 5 and + pair[1][-1] == pair[1][0] and + pair[1][-2] == pair[1][0] and + pair[1][-3] == pair[1][0])): + k = len(pair[1]) - 1 + while k > -1 and pair[1][k] == '\\': + multibackslash = not multibackslash + k -= 1 + if multibackslash: + multilinestr = pair[1][:-1] + else: + multilinestr = pair[1] + "\n" + multikey = pair[0] + else: + value, vtype = _load_value(pair[1], _dict, strictly_valid) + try: + currentlevel[pair[0]] + raise TomlDecodeError("Duplicate keys!") + except KeyError: + if multikey: + return multikey, multilinestr, multibackslash + else: + currentlevel[pair[0]] = value + except: + raise TomlDecodeError("Duplicate keys!") + + +def _load_date(val): + microsecond = 0 + tz = None + try: + if len(val) > 19: + if val[19] == '.': + microsecond = int(val[20:26]) + if len(val) > 26: + tz = TomlTz(val[26:32]) + else: + tz = TomlTz(val[19:25]) + except ValueError: + tz = None + try: + d = datetime.datetime( + int(val[:4]), int(val[5:7]), + int(val[8:10]), int(val[11:13]), + int(val[14:16]), int(val[17:19]), microsecond, tz) + except ValueError: + return None + return d + + +def _load_unicode_escapes(v, hexbytes, prefix): + hexchars = ['0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'] + skip = False + i = len(v) - 1 + while i > -1 and v[i] == '\\': + skip = not skip + i -= 1 + for hx in hexbytes: + if skip: + skip = False + i = len(hx) - 1 + while i > -1 and hx[i] == '\\': + skip = not skip + i -= 1 + v += prefix + v += hx + continue + hxb = "" + i = 0 + hxblen = 4 + if prefix == "\\U": + hxblen = 8 + while i < hxblen: + try: + if not hx[i].lower() in hexchars: + raise IndexError("This is a hack") + except IndexError: + raise TomlDecodeError("Invalid escape sequence") + hxb += hx[i].lower() + i += 1 + v += unichr(int(hxb, 16)) + v += unicode(hx[len(hxb):]) + return v + + +# Unescape TOML string values. + +# content after the \ +_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] +# What it should be replaced by +_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] +# Used for substitution +_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) + + +def _unescape(v): + """Unescape characters in a TOML string.""" + i = 0 + backslash = False + while i < len(v): + if backslash: + backslash = False + if v[i] in _escapes: + v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] + elif v[i] == '\\': + v = v[:i - 1] + v[i:] + elif v[i] == 'u' or v[i] == 'U': + i += 1 + else: + raise TomlDecodeError("Reserved escape sequence used") + continue + elif v[i] == '\\': + backslash = True + i += 1 + return v + + +def _load_value(v, _dict, strictly_valid=True): + if not v: + raise TomlDecodeError("Empty value is invalid") + if v == 'true': + return (True, "bool") + elif v == 'false': + return (False, "bool") + elif v[0] == '"': + testv = v[1:].split('"') + if testv[0] == '' and testv[1] == '': + testv = testv[2:-2] + closed = False + for tv in testv: + if tv == '': + closed = True + else: + oddbackslash = False + try: + i = -1 + j = tv[i] + while j == '\\': + oddbackslash = not oddbackslash + i -= 1 + j = tv[i] + except IndexError: + pass + if not oddbackslash: + if closed: + raise TomlDecodeError("Stuff after closed string. WTF?") + else: + closed = True + escapeseqs = v.split('\\')[1:] + backslash = False + for i in escapeseqs: + if i == '': + backslash = not backslash + else: + if i[0] not in _escapes and (i[0] != 'u' and i[0] != 'U' and + not backslash): + raise TomlDecodeError("Reserved escape sequence used") + if backslash: + backslash = False + for prefix in ["\\u", "\\U"]: + if prefix in v: + hexbytes = v.split(prefix) + v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], prefix) + v = _unescape(v) + if v[1] == '"' and (len(v) < 3 or v[1] == v[2]): + v = v[2:-2] + return (v[1:-1], "str") + elif v[0] == "'": + if v[1] == "'" and (len(v) < 3 or v[1] == v[2]): + v = v[2:-2] + return (v[1:-1], "str") + elif v[0] == '[': + return (_load_array(v, _dict), "array") + elif v[0] == '{': + inline_object = _get_empty_inline_table(_dict) + _load_inline_object(v, inline_object, _dict) + return (inline_object, "inline_object") + else: + parsed_date = _load_date(v) + if parsed_date is not None: + return (parsed_date, "date") + if not strictly_valid: + raise TomlDecodeError("Weirdness with leading zeroes or underscores" + " in your number.") + itype = "int" + neg = False + if v[0] == '-': + neg = True + v = v[1:] + elif v[0] == '+': + v = v[1:] + v = v.replace('_', '') + if '.' in v or 'e' in v or 'E' in v: + if '.' in v and v.split('.', 1)[1] == '': + raise TomlDecodeError("This float is missing digits after " + "the point") + if v[0] not in '0123456789': + raise TomlDecodeError("This float doesn't have a leading digit") + v = float(v) + itype = "float" + else: + v = int(v) + if neg: + return (0 - v, itype) + return (v, itype) + + +def _load_array(a, _dict): + atype = None + retval = [] + a = a.strip() + if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): + strarray = False + tmpa = a[1:-1].strip() + if tmpa != '' and (tmpa[0] == '"' or tmpa[0] == "'"): + strarray = True + if not a[1:-1].strip().startswith('{'): + a = a[1:-1].split(',') + else: + # a is an inline object, we must find the matching parenthesis + # to define groups + new_a = [] + start_group_index = 1 + end_group_index = 2 + in_str = False + while end_group_index < len(a[1:]): + if a[end_group_index] == '"' or a[end_group_index] == "'": + in_str = not in_str + if in_str or a[end_group_index] != '}': + end_group_index += 1 + continue + + # Increase end_group_index by 1 to get the closing bracket + end_group_index += 1 + new_a.append(a[start_group_index:end_group_index]) + + # The next start index is at least after the closing bracket, a + # closing bracket can be followed by a comma since we are in + # an array. + start_group_index = end_group_index + 1 + while (start_group_index < len(a[1:]) and + a[start_group_index] != '{'): + start_group_index += 1 + end_group_index = start_group_index + 1 + a = new_a + b = 0 + if strarray: + while b < len(a) - 1: + ab = a[b].strip() + while ab[-1] != ab[0] or (ab[0] == ab[1] == ab[2] and + ab[-2] != ab[0] and ab[-3] != ab[0]): + a[b] = a[b] + ',' + a[b + 1] + ab = a[b].strip() + if b < len(a) - 2: + a = a[:b + 1] + a[b + 2:] + else: + a = a[:b + 1] + b += 1 + else: + al = list(a[1:-1]) + a = [] + openarr = 0 + j = 0 + for i in _range(len(al)): + if al[i] == '[': + openarr += 1 + elif al[i] == ']': + openarr -= 1 + elif al[i] == ',' and not openarr: + a.append(''.join(al[j:i])) + j = i + 1 + a.append(''.join(al[j:])) + for i in _range(len(a)): + a[i] = a[i].strip() + if a[i] != '': + nval, ntype = _load_value(a[i], _dict) + if atype: + if ntype != atype: + raise TomlDecodeError("Not a homogeneous array") + else: + atype = ntype + retval.append(nval) + return retval + + +def dump(o, f): + """Writes out dict as toml to a file + + Args: + o: Object to dump into toml + f: File descriptor where the toml should be stored + + Returns: + String containing the toml corresponding to dictionary + + Raises: + TypeError: When anything other than file descriptor is passed + """ + + if not f.write: + raise TypeError("You can only dump an object to a file descriptor") + d = dumps(o) + f.write(d) + return d + + +def dumps(o, preserve=False): + """Stringifies input dict as toml + + Args: + o: Object to dump into toml + + preserve: Boolean parameter. If true, preserve inline tables. + + Returns: + String containing the toml corresponding to dict + """ + + retval = "" + addtoretval, sections = _dump_sections(o, "") + retval += addtoretval + while sections != {}: + newsections = {} + for section in sections: + addtoretval, addtosections = _dump_sections(sections[section], + section, preserve) + if addtoretval or (not addtoretval and not addtosections): + if retval and retval[-2:] != "\n\n": + retval += "\n" + retval += "[" + section + "]\n" + if addtoretval: + retval += addtoretval + for s in addtosections: + newsections[section + "." + s] = addtosections[s] + sections = newsections + return retval + + +def _dump_sections(o, sup, preserve=False): + retstr = "" + if sup != "" and sup[-1] != ".": + sup += '.' + retdict = o.__class__() + arraystr = "" + for section in o: + section = str(section) + qsection = section + if not re.match(r'^[A-Za-z0-9_-]+$', section): + if '"' in section: + qsection = "'" + section + "'" + else: + qsection = '"' + section + '"' + if not isinstance(o[section], dict): + arrayoftables = False + if isinstance(o[section], list): + for a in o[section]: + if isinstance(a, dict): + arrayoftables = True + if arrayoftables: + for a in o[section]: + arraytabstr = "\n" + arraystr += "[[" + sup + qsection + "]]\n" + s, d = _dump_sections(a, sup + qsection) + if s: + if s[0] == "[": + arraytabstr += s + else: + arraystr += s + while d != {}: + newd = {} + for dsec in d: + s1, d1 = _dump_sections(d[dsec], sup + qsection + + "." + dsec) + if s1: + arraytabstr += ("[" + sup + qsection + "." + + dsec + "]\n") + arraytabstr += s1 + for s1 in d1: + newd[dsec + "." + s1] = d1[s1] + d = newd + arraystr += arraytabstr + else: + if o[section] is not None: + retstr += (qsection + " = " + + str(_dump_value(o[section])) + '\n') + elif preserve and isinstance(o[section], InlineTableDict): + retstr += (section + " = " + _dump_inline_table(o[section])) + else: + retdict[qsection] = o[section] + retstr += arraystr + return (retstr, retdict) + + +def _dump_inline_table(section): + """Preserve inline table in its compact syntax instead of expanding + into subsection. + + https://github.com/toml-lang/toml#user-content-inline-table + """ + retval = "" + if isinstance(section, dict): + val_list = [] + for k, v in section.items(): + val = _dump_inline_table(v) + val_list.append(k + " = " + val) + retval += "{ " + ", ".join(val_list) + " }\n" + return retval + else: + return str(_dump_value(section)) + + +def _dump_value(v): + dump_funcs = { + str: lambda: _dump_str(v), + unicode: lambda: _dump_str(v), + list: lambda: _dump_list(v), + bool: lambda: str(v).lower(), + float: lambda: _dump_float(v), + datetime.datetime: lambda: v.isoformat(), + } + # Lookup function corresponding to v's type + dump_fn = dump_funcs.get(type(v)) + # Evaluate function (if it exists) else return v + return dump_fn() if dump_fn is not None else v + + +def _dump_str(v): + v = "%r" % v + if v[0] == 'u': + v = v[1:] + singlequote = v.startswith("'") + v = v[1:-1] + if singlequote: + v = v.replace("\\'", "'") + v = v.replace('"', '\\"') + v = v.replace("\\x", "\\u00") + return str('"' + v + '"') + + +def _dump_list(v): + t = [] + retval = "[" + for u in v: + t.append(_dump_value(u)) + while t != []: + s = [] + for u in t: + if isinstance(u, list): + for r in u: + s.append(r) + else: + retval += " " + str(u) + "," + t = s + retval += "]" + return retval + + +def _dump_float(v): + return "{0:.16g}".format(v).replace("e+0", "e+").replace("e-0", "e-") diff --git a/requirements_dev.txt b/requirements_dev.txt index bab94fb..41884c4 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -11,3 +11,4 @@ PyYAML==3.11 pytest==2.9.2 pytest-runner==2.11.1 setuptools<=26.1.1 +pipenv diff --git a/setup.py b/setup.py index fc119b8..2350b02 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,6 @@ "packaging", "six", "pyyaml", - "pipenv" ] # make pytest-runner a conditional requirement, per: https://pypi.org/project/pytest-runner/ @@ -60,4 +59,7 @@ test_suite='tests', tests_require=test_requirements, setup_requires=setup_requirements, + extras_require={ + 'pipenv': ["pipenv"], + } ) From be443b6c3367a7d2e307e2ce4b174c80fb27d3c1 Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:01:19 +0200 Subject: [PATCH 05/10] remove unused dev dependencies --- requirements_dev.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 41884c4..2d71672 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,5 @@ pip==8.1.2 -bumpversion==0.5.3 wheel==0.29.0 -watchdog==0.8.3 flake8==2.6.0 tox==2.3.1 coverage==4.1 From 1612f444f924216d87c7923f188a2e6e6635b692 Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:03:51 +0200 Subject: [PATCH 06/10] add 3.7, remove 3.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2350b02..41c88d3 100644 --- a/setup.py +++ b/setup.py @@ -52,9 +52,9 @@ "Programming Language :: Python :: 2", 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', ], test_suite='tests', tests_require=test_requirements, From 779c853dae870006ad3d290f8d91d0e8b37ad384 Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:08:45 +0200 Subject: [PATCH 07/10] new release, 0.4 --- HISTORY.rst | 7 +++++++ dparse/__init__.py | 4 ++-- setup.py | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c9d432d..9cb5d7f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,13 @@ History ======= +0.4.0 (2018-04-06) +------------------ + +* pipenv is now an optional dependency that's only used when updating a Pipfile. Install it with dparse[pipenv] +* Added support for invalid toml Pipfiles (thanks @pombredanne) + + 0.3.0 (2018-03-01) ------------------ diff --git a/dparse/__init__.py b/dparse/__init__.py index 824cca1..21cb434 100644 --- a/dparse/__init__.py +++ b/dparse/__init__.py @@ -3,7 +3,7 @@ """Top-level package for Dependency Parser.""" __author__ = """Jannis Gebauer""" -__email__ = 'support@pyup.io' -__version__ = '0.3.0' +__email__ = 'jay@pyup.io' +__version__ = '0.4.0' from .parser import parse diff --git a/setup.py b/setup.py index 41c88d3..9b09090 100644 --- a/setup.py +++ b/setup.py @@ -32,11 +32,11 @@ setup( name='dparse', - version='0.3.0', + version='0.4.0', description="A parser for Python dependency files", long_description=readme + '\n\n' + history, author="Jannis Gebauer", - author_email='ja.geb@me.com', + author_email='jay@pyup.io', url='https://github.com/jayfk/dparse', packages=find_packages(include=['dparse']), include_package_data=True, From 6320192e5990f61e8ace23d1e08ba844b70cf711 Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:10:26 +0200 Subject: [PATCH 08/10] update readme on how to install the pipenv extra --- README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3a0a06c..ca4ee87 100644 --- a/README.rst +++ b/README.rst @@ -46,7 +46,7 @@ Supported Files +------------------+------------+-----------+ | setup.py | no (# 2_) | no (# 2_) | +------------------+------------+-----------+ -| zc.buildout | no (# 3_) | no (# 3_) | +| zc.buildout | no (# 3_) | no (# 3_) | +------------------+------------+-----------+ | setup.cfg | no (# 4_) | no (# 4_) | +------------------+------------+-----------+ @@ -65,6 +65,12 @@ To install dparse, run: $ pip install dparse +If you want to update Pipfiles, install the pipenv extra: + +.. code-block:: console + + $ pip install dparse[pipenv] + ***** Usage ***** From 19bdc0e572b303b203122b46d6ff96829996baf2 Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:17:30 +0200 Subject: [PATCH 09/10] remove pytest cache --- .pytest_cache/v/cache/lastfailed | 3 -- .pytest_cache/v/cache/nodeids | 49 -------------------------------- 2 files changed, 52 deletions(-) delete mode 100644 .pytest_cache/v/cache/lastfailed delete mode 100644 .pytest_cache/v/cache/nodeids diff --git a/.pytest_cache/v/cache/lastfailed b/.pytest_cache/v/cache/lastfailed deleted file mode 100644 index 609de55..0000000 --- a/.pytest_cache/v/cache/lastfailed +++ /dev/null @@ -1,3 +0,0 @@ -{ - "tests/test_updater.py::test_update_pipfile": true -} \ No newline at end of file diff --git a/.pytest_cache/v/cache/nodeids b/.pytest_cache/v/cache/nodeids deleted file mode 100644 index ba00e17..0000000 --- a/.pytest_cache/v/cache/nodeids +++ /dev/null @@ -1,49 +0,0 @@ -[ - "tests/test_dependencies.py::test_dependency_serialize", - "tests/test_dependencies.py::test_dependency_deserialize", - "tests/test_dependencies.py::test_dependency_file_serialize", - "tests/test_dependencies.py::test_dependency_file_deserialize", - "tests/test_dependencies.py::test_parser_class", - "tests/test_parse.py::test_requirements_with_invalid_requirement", - "tests/test_parse.py::test_tox_ini_with_invalid_requirement", - "tests/test_parse.py::test_conda_file_with_invalid_requirement", - "tests/test_parse.py::test_conda_file_invalid_yml", - "tests/test_parse.py::test_conda_file_marked_line", - "tests/test_parse.py::test_tox_ini_marked_line", - "tests/test_parse.py::test_resolve_file", - "tests/test_parse.py::test_index_server", - "tests/test_parse.py::test_requirements_package_with_index_server", - "tests/test_parse.py::test_requirements_parse_empty_line", - "tests/test_parse.py::test_requirements_parse_unsupported_line_start", - "tests/test_parse.py::test_file_resolver", - "tests/test_parse.py::test_is_marked_file", - "tests/test_parse.py::test_is_marked_line", - "tests/test_parse.py::test_pipfile", - "tests/test_parse.py::test_pipfile_lock", - "tests/test_updater.py::test_update_tox_ini", - "tests/test_updater.py::test_update_conda_yml", - "tests/test_updater.py::test_update_requirements_multispace", - "tests/test_updater.py::test_update_requirements_compatible", - "tests/test_updater.py::test_update_requirements_compatible_matching_latest", - "tests/test_updater.py::test_update_requirements_contains_correct_sep_char", - "tests/test_updater.py::test_update_requirements_with_hashes", - "tests/test_updater.py::test_update_requirements_with_hashes_and_comment_and_env_markers", - "tests/test_updater.py::test_update_requirements_with_hashes_and_comment_inline", - "tests/test_updater.py::test_update_requirements_with_hash_and_space_separator", - "tests/test_updater.py::test_update_requirements_with_hash_and_comment_and_env_markers_inline", - "tests/test_updater.py::test_update_requirements_with_hash_inline", - "tests/test_updater.py::test_update_requirements_with_env_markers", - "tests/test_updater.py::test_update_requirements_with_env_markers_and_comment", - "tests/test_updater.py::test_update_requirements_with_extras", - "tests/test_updater.py::test_update_requirements_with_tabs", - "tests/test_updater.py::test_update_requirements_with_plus", - "tests/test_updater.py::test_update_requirements_line_endings", - "tests/test_updater.py::test_update_requirements_simple_pinned", - "tests/test_updater.py::test_update_requirements_simple_unpinned", - "tests/test_updater.py::test_update_requirements_simple_unpinned_with_comment", - "tests/test_updater.py::test_update_requirements_cookiecutter_template", - "tests/test_updater.py::test_update_requirements_with_double_package_name", - "tests/test_updater.py::test_update_requirements_ranged", - "tests/test_updater.py::test_update_requirements_unfinished_line", - "tests/test_updater.py::test_update_pipfile" -] \ No newline at end of file From 0cd5aa7eb1f78c39da78b6c63dde6b49a1732cd2 Mon Sep 17 00:00:00 2001 From: Jannis Gebauer Date: Fri, 6 Apr 2018 13:41:53 +0200 Subject: [PATCH 10/10] fixed a packaging error, new release 0.4.1 --- HISTORY.rst | 5 +++++ MANIFEST.in | 1 + dparse/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9cb5d7f..c7f98e2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,11 @@ History ======= +0.4.1 (2018-04-06) +------------------ + +* Fixed a packaging error. + 0.4.0 (2018-04-06) ------------------ diff --git a/MANIFEST.in b/MANIFEST.in index 292d6dd..f95d9b0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ recursive-exclude * __pycache__ recursive-exclude * *.py[co] recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif +recursive-include dparse *.py diff --git a/dparse/__init__.py b/dparse/__init__.py index 21cb434..4b233e9 100644 --- a/dparse/__init__.py +++ b/dparse/__init__.py @@ -4,6 +4,6 @@ __author__ = """Jannis Gebauer""" __email__ = 'jay@pyup.io' -__version__ = '0.4.0' +__version__ = '0.4.1' from .parser import parse diff --git a/setup.py b/setup.py index 9b09090..15f9c41 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( name='dparse', - version='0.4.0', + version='0.4.1', description="A parser for Python dependency files", long_description=readme + '\n\n' + history, author="Jannis Gebauer",