Vendor python-forntmatter completely

We used to vendor parts of python frontmatter and also use it as
a dependency which was problematic. This vendors python-frontmatter
fully and only uses the code needed.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

requirements.txt

@@ -56,7 +56,6 @@ pygmars==0.7.0
 Pygments==2.12.0
 pymaven-patch==0.3.0
 pyparsing==3.0.9
-python-frontmatter==1.0.0
 pytz==2022.1
 PyYAML==6.0
 rdflib==6.2.0

setup.cfg

@@ -101,7 +101,6 @@ install_requires =
     pygmars >= 0.7.0
     pygments
     pymaven_patch >= 0.2.8
-    python-frontmatter >= 1.0.0
     requests >= 2.7.0
     saneyaml >= 0.5.2
     spdx_tools >= 0.7.0a3

src/licensedcode/frontmatter.py

@@ -13,26 +13,63 @@
 import saneyaml
 import re
 
-from frontmatter import detect_format
-from frontmatter import handlers
-from frontmatter import Post as FrontmatterPost
-from frontmatter.default_handlers import BaseHandler
-from frontmatter.util import u
 
-from licensedcode.tokenize import query_lines
+DEFAULT_POST_TEMPLATE = """\
+{start_delimiter}
+{metadata}
+{end_delimiter}
 
+{content}
+"""
 
 
-class SaneYAMLHandler(BaseHandler):
+class SaneYAMLHandler:
     """
     Load and export YAML metadata. .
 
-    This is similar to the frontmatter.default_handlers.YAMLHandler but
-    is using nexB/saneyaml instead of pyyaml.
+    This is similar to the original frontmatter.default_handlers.YAMLHandler
+    but is using nexB/saneyaml instead of pyyaml.
     """
     FM_BOUNDARY = re.compile(r"^-{3,}\s*$", re.MULTILINE)
     START_DELIMITER = END_DELIMITER = "---"
 
+    def __init__(self):
+        self.FM_BOUNDARY = self.FM_BOUNDARY
+        self.START_DELIMITER = self.START_DELIMITER
+        self.END_DELIMITER = self.END_DELIMITER
+
+    def detect(self, text):
+        """
+        Decide whether this handler can parse the given ``text``,
+        and return True or False.
+        """
+        if self.FM_BOUNDARY.match(text):
+            return True
+        return False
+
+    def split(self, text):
+        """
+        Split text into frontmatter and content
+        """
+        _, fm, content = self.FM_BOUNDARY.split(text, 2)
+        return fm, content
+
+    def format(self, content, metadata, **kwargs):
+        """
+        Turn a post into a string, used in ``frontmatter.dumps``
+        """
+        start_delimiter = kwargs.pop("start_delimiter", self.START_DELIMITER)
+        end_delimiter = kwargs.pop("end_delimiter", self.END_DELIMITER)
+
+        metadata = self.export(metadata, **kwargs)
+
+        return DEFAULT_POST_TEMPLATE.format(
+            metadata=metadata,
+            content=content,
+            start_delimiter=start_delimiter,
+            end_delimiter=end_delimiter,
+        ).strip()
+
     def load(self, fm, **kwargs):
         """
         Parse YAML front matter.
@@ -44,42 +81,34 @@ def export(self, metadata, **kwargs):
         Export metadata as YAML.
         """
         metadata = saneyaml.dump(metadata, indent=4, encoding='utf-8', **kwargs).strip()
-        return u(metadata)  # ensure unicode
+        return return_unicode(metadata)  # ensure unicode
 
 
-def get_rule_text(location=None, text=None):
-    """
-    Return the rule ``text`` prepared for indexing.
-    ###############
-    # IMPORTANT: we use the same process as used to load query text for symmetry
-    ###############
-    """
-    numbered_lines = query_lines(location=location, query_string=text, plain_text=True)
-    return '\n'.join(l.strip() for _, l in numbered_lines)
+def return_unicode(text, encoding="utf-8"):
+    "Return unicode text, no matter what"
+
+    if isinstance(text, bytes):
+        text = text.decode(encoding)
 
+    # it's already unicode
+    text = text.replace("\r\n", "\n")
+    return text
 
-def parse_frontmatter(text, encoding="utf-8", handler=None, **defaults):
+
+def parse_frontmatter(text, encoding="utf-8", handler=SaneYAMLHandler(), **defaults):
     """
     Parse text with frontmatter, return metadata and content.
     Pass in optional metadata defaults as keyword args.
 
     If frontmatter is not found, returns an empty metadata dictionary
     (or defaults) and original text content.
-
-    This is similar to the frontmatter.parse but is using `get_rule_text`
-    to use the same process as loading quary text for symmetry.
     """
     # ensure unicode first
-    text = u(text, encoding).strip()
+    text = return_unicode(text, encoding)
 
     # metadata starts with defaults
     metadata = defaults.copy()
 
-    # this will only run if a handler hasn't been set higher up
-    handler = handler or detect_format(text, handlers)
-    if handler is None:
-        return metadata, text
-
     # split on the delimiters
     try:
         fm, content = handler.split(text)
@@ -92,57 +121,27 @@ def parse_frontmatter(text, encoding="utf-8", handler=None, **defaults):
     if isinstance(fm, dict):
         metadata.update(fm)
 
-    text = get_rule_text(text=content)
+    return content, metadata
 
-    return metadata, text
 
-
-def loads_frontmatter(text, encoding="utf-8", handler=None, **defaults):
+def load_frontmatter(fd, encoding="utf-8", **defaults):
     """
-    Parse text (binary or unicode) and return a :py:class:`post <frontmatter.Post>`.
-
-    This is similar to the frontmatter.loads but is using the `parse`
-    function defined above.
-    """
-    text = u(text, encoding)
-    handler = handler or detect_format(text, handlers)
-    metadata, content = parse_frontmatter(text, encoding, handler, **defaults)
-    return FrontmatterPost(content, handler, **metadata)
-
-
-def load_frontmatter(fd, encoding="utf-8", handler=None, **defaults):
-    """
-    Load and parse a file-like object or filename,
-    return a :py:class:`post <frontmatter.Post>`.
-
-    This is similar to the frontmatter.load but is using the `loads`
-    function defined above.
+    Load and parse a file-like object or filename, and return
+    `content` and `metadata` with the text and the frontmatter metadata.
     """
     if hasattr(fd, "read"):
         text = fd.read()
 
     else:
         with codecs.open(fd, "r", encoding) as f:
             text = f.read()
-
-    handler = handler or detect_format(text, handlers)
-    return loads_frontmatter(text, encoding, handler, **defaults)
+    
+    text = return_unicode(text, encoding)
+    return parse_frontmatter(text, encoding, **defaults)
 
 
-def dumps_frontmatter(post, handler=None, **kwargs):
+def dumps_frontmatter(content, metadata, handler=SaneYAMLHandler(), **kwargs):
     """
-    Serialize a :py:class:`post <frontmatter.Post>` to a string and return text.
-    This always returns unicode text, which can then be encoded.
-
-    Passing ``handler`` will change how metadata is turned into text. A handler
-    passed as an argument will override ``post.handler``, with
-    :py:class:`SaneYAMLHandler <frontmatter.SaneYAMLHandler>` used as
-    a default.
-
-    This is similar to the frontmatter.dumps but is using the `SaneYAMLHandler`
-    defined above as default instead of frontmatter.default_handlers.YAMLHandler.
+    Create a string and return the text from `content` and `metadata`.
     """
-    if handler is None:
-        handler = getattr(post, "handler", None) or SaneYAMLHandler()
-
-    return handler.format(post, **kwargs)
+    return handler.format(content, metadata, **kwargs)

src/licensedcode/models.py

@@ -33,18 +33,16 @@
 from licensedcode import MIN_MATCH_HIGH_LENGTH
 from licensedcode import MIN_MATCH_LENGTH
 from licensedcode import SMALL_RULE
-from licensedcode.frontmatter import SaneYAMLHandler
-from licensedcode.frontmatter import FrontmatterPost
 from licensedcode.frontmatter import dumps_frontmatter
 from licensedcode.frontmatter import load_frontmatter
-from licensedcode.frontmatter import get_rule_text
 from licensedcode.languages import LANG_INFO as known_languages
 from licensedcode.spans import Span
 from licensedcode.tokenize import index_tokenizer
 from licensedcode.tokenize import index_tokenizer_with_stopwords
 from licensedcode.tokenize import key_phrase_tokenizer
 from licensedcode.tokenize import KEY_PHRASE_OPEN
 from licensedcode.tokenize import KEY_PHRASE_CLOSE
+from licensedcode.tokenize import query_lines
 
 """
 Reference License and license Rule structures persisted as a combo of a YAML
@@ -427,17 +425,9 @@ def dump(self, licenses_data_dir):
          - the license data as YAML frontmatter
          - the license text
         """
-
-        def write(location, byte_string):
-            # we write as binary because rules and licenses texts and data are
-            # UTF-8-encoded bytes
-            with io.open(location, 'wb') as of:
-                of.write(byte_string)
-
         metadata = self.to_dict(include_builtin=False)
         content = self.text
-        rule_post = FrontmatterPost(content=content, handler=SaneYAMLHandler(), **metadata)
-        output = dumps_frontmatter(post=rule_post)
+        output = dumps_frontmatter(content=content, metadata=metadata)
         license_file = self.license_file(licenses_data_dir=licenses_data_dir)
         with open(license_file, 'w') as of:
             of.write(output)
@@ -449,16 +439,15 @@ def load(self, license_file, check_consistency=True):
         Unknown fields are ignored and not bound to the License object.
         """
         try:
-            post = load_frontmatter(license_file)
-            data = post.metadata
+            content, data = load_frontmatter(license_file)
             if check_consistency:
                 if not data:
                     raise InvalidLicense(
                         f'Cannot load License with empty YAML frontmatter: '
                         f'{self}: file://{license_file}'
                     )
 
-            if not post.content:
+            if not content:
                 if check_consistency:
                     if not any(
                         attribute in data
@@ -472,7 +461,7 @@ def load(self, license_file, check_consistency=True):
 
                 self.text = ''
             else:
-                self.text = post.content.lstrip("\n")
+                self.text = content.lstrip("\n")
 
             for k, v in data.items():
                 if k == 'minimum_coverage':
@@ -1043,6 +1032,17 @@ def build_rule_from_license(license_obj):
         return rule
 
 
+def get_rule_text(location=None, text=None):
+    """
+    Return the rule ``text`` prepared for indexing.
+    ###############
+    # IMPORTANT: we use the same process as used to load query text for symmetry
+    ###############
+    """
+    numbered_lines = query_lines(location=location, query_string=text, plain_text=True)
+    return '\n'.join(l.strip() for _, l in numbered_lines)
+
+
 def get_all_spdx_keys(licenses_db):
     """
     Return an iterable of SPDX license keys collected from a `licenses_db`
@@ -2084,8 +2084,7 @@ def dump(self, rules_data_dir):
 
         metadata = self.to_dict()
         content = self.text
-        rule_post = FrontmatterPost(content=content, handler=SaneYAMLHandler(), **metadata)
-        output = dumps_frontmatter(post=rule_post)
+        output = dumps_frontmatter(content=content, metadata=metadata)
         with open(rule_file, 'w') as of:
             of.write(output)
 
@@ -2096,15 +2095,14 @@ def load(self, rule_file, with_checks=True):
         Optionally check for consistency if ``with_checks`` is True.
         """
         try:
-            post = load_frontmatter(rule_file)
-            data = post.metadata
-            if not post.content:
+            content, data = load_frontmatter(rule_file)
+            if not content:
                 raise InvalidRule(
                     f'Cannot load rule with empty text: '
                     f'{self}: file://{rule_file}'
                 )
 
-            self.text = post.content.lstrip()
+            self.text = content.lstrip()
 
         except Exception as e:
             print('#############################')