Enable generation of copyright test data files

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/licensedcode/data/licenses/mtll.LICENSE

@@ -10,6 +10,9 @@ notes: |
     derived from the Apache Software License v1.1, but it is modified in
     several ways. It is Free, but GPL-incompatible.
 spdx_license_key: MTLL
+ignorable_authors:
+  - at the University of Notre Dame, the Pervasive Technology Labs at Indiana University, and
+    Dresden University of Technology
 ---
 
 Software License for MTL

src/licensedcode/data/licenses/nero-eula.LICENSE

@@ -15,9 +15,9 @@ ignorable_copyrights:
     - Copyright (c) 1991-2, RSA Data Security, Inc.
     - Copyright (c) 1992-1999 Dolby Laboratories, Inc.
     - Copyright (c) 1996-2012 Nero AG and its licensors
+    - Copyright (c) 1999-2002. Microsoft Corporation
     - Copyright (c) 2008 Microsoft Corporation
     - Copyright 2011 Dolby Laboratories
-    - Media Technologies. Copyright (c) 1999-2002. Microsoft Corporation
     - copyright (c) 1998-2005 The OpenSSL Project
     - copyright (c) 2000-2008 Gracenote
     - copyright (c) 2000-2008 Gracenote. Gracenote Software
@@ -33,7 +33,6 @@ ignorable_holders:
     - Dr Brian Gladman, Worcester, UK.
     - Gracenote
     - Gracenote. Gracenote Software
-    - Media Technologies. Microsoft Corporation
     - Microsoft Corporation
     - Nero AG and its licensors
     - Nero AG.
@@ -402,4 +401,4 @@ Other product and brand names may be trademarks of their respective owners and d
 www.nero.com 
 
 If you have any questions concerning this Agreement contact us via legal@nero.com. 
-© 2012 Nero AG.  All rights reserved.
+© 2012 Nero AG.  All rights reserved.

tests/cluecode/cluecode_test_utils.py

@@ -8,6 +8,7 @@
 #
 
 import io
+import os
 from itertools import chain
 from os import path
 
@@ -23,7 +24,6 @@
 from cluecode.copyrights import Detection
 from scancode_config import REGEN_TEST_FIXTURES
 
-
 """
 Data-driven Copyright test utilities.
 """
@@ -129,19 +129,56 @@ def dump(self, check_exists=False):
 def load_copyright_tests(test_dir=test_env.test_data_dir, generate_missing=False):
     """
     Yield an iterable of CopyrightTest loaded from test data files in `test_dir`.
+    If `generate_missing` is True, generate missing YAML data files with a default test template.
     """
-    test_dirs = (path.join(test_dir, td) for td in
-        ('copyrights', 'ics', 'holders', 'authors', 'years', 'generated'))
+    test_dirs = [
+        path.join(test_dir, td)
+        for td in
+        ('copyrights', 'ics', 'holders', 'authors', 'years', 'generated')
+    ]
+
+    if generate_missing:
+        for td in test_dirs:
+            create_missing_test_data_files(test_dir=td)
 
-    all_test_files = chain.from_iterable(
-        get_test_file_pairs(td, generate_missing=generate_missing)
-        for td in test_dirs
-    )
+    all_test_files = chain.from_iterable(get_test_file_pairs(td) for td in test_dirs)
 
     for data_file, test_file in all_test_files:
         yield CopyrightTest(data_file, test_file)
 
 
+empty_data_file = """what:
+  - copyrights
+  - holders
+  - holders_summary
+  - authors
+"""
+
+
+def create_missing_test_data_files(test_dir):
+    """
+    Create missing copyright test data files in a `test_dir` directory.
+
+    Each test consist of a pair of files:
+    - a test file.
+    - a data file with the same name as a test file and a '.yml' extension added.
+    """
+    for top, _, files in os.walk(test_dir):
+        for tfile in files:
+            if tfile.endswith('~'):
+                continue
+            file_path = path.abspath(path.join(top, tfile))
+
+            if tfile.endswith('.yml'):
+                data_file_path = file_path
+            else:
+                data_file_path = file_path + '.yml'
+
+            if not path.exists(data_file_path):
+                with open(data_file_path, "w") as tfo:
+                    tfo.write(empty_data_file)
+
+
 def as_sorted_mapping(counter):
     """
     Return a list of ordered mapping of {value:val, count:cnt} built from a

tests/cluecode/test_copyrights.py

@@ -31,8 +31,8 @@ class TestCopyrightDataDriven(FileBasedTesting):
     pass
 
 
-build_tests(copyright_tests=load_copyright_tests(
-    generate_missing=REGEN_TEST_FIXTURES),
+build_tests(
+    copyright_tests=load_copyright_tests(generate_missing=REGEN_TEST_FIXTURES),
     clazz=TestCopyrightDataDriven,
     regen=REGEN_TEST_FIXTURES,
 )