Make license texts YAML safe

* Makes license reference texts YAML safe
* Makes matched texts YAML safe

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -436,6 +436,7 @@ def dict_fields(attr, value):
                     include_text=include_text,
                     license_text_diagnostics=license_text_diagnostics,
                     whole_lines=whole_lines,
+                    yaml_safe=True,
                 )
             )
 

src/licensedcode/match.py

@@ -17,6 +17,7 @@
 from licensedcode import MAX_DIST
 from licensedcode import SMALL_RULE
 from licensedcode import query
+from licensedcode.models import get_yaml_safe_text
 from licensedcode.spans import Span
 from licensedcode.stopwords import STOPWORDS
 from licensedcode.tokenize import index_tokenizer
@@ -718,6 +719,7 @@ def itokens_hash(self, idx):
     # location at once to avoid reprocessing many times the original text
     def matched_text(
         self,
+        yaml_safe=False,
         whole_lines=False,
         highlight=True,
         highlight_matched='{}',
@@ -744,7 +746,7 @@ def matched_text(
         if whole_lines and query.has_long_lines:
             whole_lines = False
 
-        return ''.join(get_full_matched_text(
+        matched_text = ''.join(get_full_matched_text(
             match=self,
             location=query.location,
             query_string=query.query_string,
@@ -756,26 +758,32 @@ def matched_text(
             _usecache=_usecache
         )).rstrip()
 
+        if yaml_safe:
+            return get_yaml_safe_text(text=matched_text)
+        else:
+            return matched_text
+
     def to_dict(
         self,
         license_url_template=SCANCODE_LICENSEDB_URL,
         spdx_license_url=SPDX_LICENSE_URL,
         include_text=False,
         license_text_diagnostics=False,
         whole_lines=True,
+        yaml_safe=False,
     ):
         """
         Return a "result" scan data built from a LicenseMatch object.
         """
         matched_text = None
         if include_text:
             if license_text_diagnostics:
-                matched_text = self.matched_text(whole_lines=False, highlight=True)
+                matched_text = self.matched_text(whole_lines=False, highlight=True, yaml_safe=yaml_safe)
             else:
                 if whole_lines:
-                    matched_text = self.matched_text(whole_lines=True, highlight=False)
+                    matched_text = self.matched_text(whole_lines=True, highlight=False, yaml_safe=yaml_safe)
                 else:
-                    matched_text = self.matched_text(whole_lines=False, highlight=False)
+                    matched_text = self.matched_text(whole_lines=False, highlight=False, yaml_safe=yaml_safe)
 
         result = {}
 

src/licensedcode/models.py

@@ -469,6 +469,7 @@ def to_reference(self):
         data['scancode_url'] = self.scancode_url
         data['licensedb_url'] = self.licensedb_url
         data['spdx_url'] = self.spdx_url
+        data['text'] = get_yaml_safe_text(text=data['text'])
         return data
 
     def dump(self, licenses_data_dir):
@@ -1867,7 +1868,7 @@ def to_reference(self):
         data['ignorable_authors'] = self.ignorable_authors
         data['ignorable_urls'] = self.ignorable_urls
         data['ignorable_emails'] = self.ignorable_emails
-        data['text'] = self.text
+        data['text'] = get_yaml_safe_text(text=self.text)
         return data
 
     def to_dict(self, include_text=False):
@@ -1969,6 +1970,28 @@ def as_int(num):
     return num
 
 
+def get_yaml_safe_text(text):
+    """
+    Given a `text` return a YAML safe version of the text
+    after removing any left-leading whitespace, empty lines and
+    lines with only whitespace.
+    """
+    if not text:
+        return
+
+    lines = text.split("\n")
+    safe_lines = []
+
+    for line in lines:
+        if line and not line.isspace():
+            safe_lines.append(
+                line.lstrip()
+            )
+
+    safe_text = "\n".join(safe_lines)
+    return safe_text
+
+
 @attr.s(slots=True)
 class Rule(BasicRule):
     """