Remove references to summary in tallies.py

    * Remove redundant code in summarizer.py

Reference: #2842
Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/summarycode/copyright_tallies.py

@@ -227,20 +227,20 @@ def tally_persons(texts):
     Return a list of mapping of {value:string, count:int} given a
     list of holders strings or Text() objects.
     """
-    texts = []
+    texts_to_tally = []
     no_detection_counter = 0
     for text in texts:
         if not text:
             no_detection_counter += 1
             continue
         # Keep Text objects as-is
         if isinstance(text, Text):
-            texts.append(text)
+            texts_to_tally.append(text)
         else:
             cano = canonical_holder(text)
-            texts.append(Text(cano, cano))
+            texts_to_tally.append(Text(cano, cano))
 
-    counter = tally(texts)
+    counter = tally(texts_to_tally)
 
     if no_detection_counter:
         counter[None] = no_detection_counter

src/summarycode/summarizer.py

@@ -12,19 +12,19 @@
 import attr
 import fingerprints
 from commoncode.cliutils import POST_SCAN_GROUP, PluggableCommandLineOption
-from plugincode.post_scan import PostScanPlugin, post_scan_impl
 from license_expression import Licensing
+from plugincode.post_scan import PostScanPlugin, post_scan_impl
+
 from cluecode.copyrights import CopyrightDetector
 from packagedcode.utils import combine_expressions
-from summarycode.copyright_summary import canonical_holder
 from summarycode.score import (compute_license_score,
                                get_field_values_from_codebase_resources,
                                unique)
-from summarycode.utils import (get_resource_summary, set_resource_summary,
-                               sorted_counter)
+from summarycode.tallies import compute_codebase_tallies
+
 
 # Tracing flags
-TRACE = False
+TRACE = True
 TRACE_LIGHT = False
 
 
@@ -72,11 +72,11 @@ def is_enabled(self, summary, **kwargs):
     def process_codebase(self, codebase, summary, **kwargs):
         if TRACE_LIGHT: logger_debug('ScanSummary:process_codebase')
 
-        # Get summary data
-        summary = summarize_codebase(codebase, keep_details=False, **kwargs)
-        license_expressions_summary = summary.get('license_expressions') or []
-        holders_summary = summary.get('holders') or []
-        programming_language_summary = summary.get('programming_language') or []
+        # Get tallies
+        tallies = compute_codebase_tallies(codebase, keep_details=False, **kwargs)
+        license_expressions_tallies = tallies.get('license_expressions') or []
+        holders_tallies = tallies.get('holders') or []
+        programming_language_tallies = tallies.get('programming_language') or []
 
         # Get Package data from key files
         key_files_package_data = get_field_values_from_codebase_resources(
@@ -100,15 +100,15 @@ def process_codebase(self, codebase, summary, **kwargs):
             # If we did not get a declared license expression from detected
             # package data, then we use the results from `compute_license_score`
             scoring_elements, declared_license_expression = compute_license_score(codebase)
-        other_license_expressions = remove_from_summary(declared_license_expression, license_expressions_summary)
+        other_license_expressions = remove_from_tallies(declared_license_expression, license_expressions_tallies)
 
         if not declared_holder:
-            declared_holder = get_declared_holder(codebase, holders_summary)
-        other_holders = remove_from_summary(declared_holder, holders_summary)
+            declared_holder = get_declared_holder(codebase, holders_tallies)
+        other_holders = remove_from_tallies(declared_holder, holders_tallies)
 
         if not primary_language:
-            primary_language = get_primary_language(programming_language_summary)
-        other_languages = remove_from_summary(primary_language, programming_language_summary)
+            primary_language = get_primary_language(programming_language_tallies)
+        other_languages = remove_from_tallies(primary_language, programming_language_tallies)
 
         # Save summary info to codebase
         codebase.attributes.summary['declared_license_expression'] = declared_license_expression
@@ -120,215 +120,23 @@ def process_codebase(self, codebase, summary, **kwargs):
         codebase.attributes.summary['other_languages'] = other_languages
 
 
-def remove_from_summary(entry, summary_data):
+def remove_from_tallies(entry, tallies):
     """
-    Return an list containing the elements of `summary_data`, without `entry`
+    Return an list containing the elements of `tallies`, without `entry`
     """
-    pruned_summary_data = []
-    for s in summary_data:
+    pruned_tallies = []
+    for t in tallies:
         if (
-            isinstance(entry, dict) and s == entry
-            or isinstance(entry, (list, tuple)) and s in entry
-            or s.get('value') == entry
+            isinstance(entry, dict) and t == entry
+            or isinstance(entry, (list, tuple)) and t in entry
+            or t.get('value') == entry
         ):
             continue
-        pruned_summary_data.append(s)
-    return pruned_summary_data
-
-
-def summarize_codebase(codebase, keep_details, **kwargs):
-    """
-    Summarize a scan at the codebase level for available scans.
-
-    If `keep_details` is True, also keep file and directory details in the
-    `summary` file attribute for every file and directory.
-
-    If `legacy` is True, summarize copyrights, authors, programming languages,
-    and packages.
-    """
-    from summarycode.copyright_summary import holder_summarizer
-
-    attrib_summarizers = [
-        ('license_expressions', license_summarizer),
-        ('holders', holder_summarizer),
-        ('programming_language', language_summarizer),
-    ]
-
-    # find which attributes are available for summarization by checking the root
-    # resource
-    root = codebase.root
-    summarizers = [s for a, s in attrib_summarizers if hasattr(root, a)]
-    if TRACE: logger_debug('summarize_codebase with summarizers:', summarizers)
-
-    # collect and set resource-level summaries
-    for resource in codebase.walk(topdown=False):
-        children = resource.children(codebase)
-
-        for summarizer in summarizers:
-            _summary_data = summarizer(resource, children, keep_details=keep_details)
-            if TRACE: logger_debug('summary for:', resource.path, 'after summarizer:', summarizer, 'is:', _summary_data)
-
-        codebase.save_resource(resource)
-
-    # set the summary from the root resource at the codebase level
-    if keep_details:
-        summary = root.summary
-    else:
-        summary = root.extra_data.get('summary', {})
-
-    if TRACE: logger_debug('codebase summary:', summary)
-
-    return summary
-
-
-def license_summarizer(resource, children, keep_details=False):
-    """
-    Populate a license_expressions list of mappings such as
-        {value: "expression", count: "count of occurences"}
-    sorted by decreasing count.
-    """
-    LIC_EXP = 'license_expressions'
-    license_expressions = []
-
-    # Collect current data
-    lic_expressions = getattr(resource, LIC_EXP  , [])
-    if not lic_expressions and resource.is_file:
-        # also count files with no detection
-        license_expressions.append(None)
-    else:
-        license_expressions.extend(lic_expressions)
-
-    # Collect direct children expression summary
-    for child in children:
-        child_summaries = get_resource_summary(child, key=LIC_EXP, as_attribute=keep_details) or []
-        for child_summary in child_summaries:
-            # TODO: review this: this feels rather weird
-            child_sum_val = child_summary.get('value')
-            if child_sum_val:
-                values = [child_sum_val] * child_summary['count']
-                license_expressions.extend(values)
-
-    # summarize proper
-    licenses_counter = summarize_licenses(license_expressions)
-    summarized = sorted_counter(licenses_counter)
-    set_resource_summary(resource, key=LIC_EXP, value=summarized, as_attribute=keep_details)
-    return summarized
-
-
-def summarize_licenses(license_expressions):
-    """
-    Given a list of license expressions, return a mapping of {expression: count
-    of occurences}
-    """
-    # TODO: we could normalize and/or sort each license_expression before
-    # summarization and consider other equivalence or containment checks
-    return Counter(license_expressions)
-
-
-def language_summarizer(resource, children, keep_details=False):
-    """
-    Populate a programming_language summary list of mappings such as
-        {value: "programming_language", count: "count of occurences"}
-    sorted by decreasing count.
-    """
-    PROG_LANG = 'programming_language'
-    languages = []
-    prog_lang = getattr(resource, PROG_LANG , [])
-    if not prog_lang:
-        if resource.is_file:
-            # also count files with no detection
-            languages.append(None)
-    else:
-        languages.append(prog_lang)
-
-    # Collect direct children expression summaries
-    for child in children:
-        child_summaries = get_resource_summary(child, key=PROG_LANG, as_attribute=keep_details) or []
-        for child_summary in child_summaries:
-            child_sum_val = child_summary.get('value')
-            if child_sum_val:
-                values = [child_sum_val] * child_summary['count']
-                languages.extend(values)
-
-    # summarize proper
-    languages_counter = summarize_languages(languages)
-    summarized = sorted_counter(languages_counter)
-    set_resource_summary(resource, key=PROG_LANG, value=summarized, as_attribute=keep_details)
-    return summarized
-
-
-def summarize_languages(languages):
-    """
-    Given a list of languages, return a mapping of {language: count
-    of occurences}
-    """
-    # TODO: consider aggregating related langauges (C/C++, etc)
-    return Counter(languages)
-
-
-SUMMARIZABLE_ATTRS = set([
-    'license_expressions',
-    'copyrights',
-    'holders',
-    'authors',
-    'programming_language',
-    # 'packages',
-])
-
-
-def add_files(packages, resource):
-    """
-    Update in-place every package mapping in the `packages` list by updating or
-    creating the the "files" attribute from the `resource`. Yield back the
-    packages.
-    """
-    for package in packages:
-        files = package['files'] = package.get('files') or []
-        fil = resource.to_dict(skinny=True)
-        if fil not in files:
-            files.append(fil)
-        yield package
-
-
-def package_summarizer(resource, children, keep_details=False):
-    """
-    Populate a packages summary list of packages mappings.
-
-    Note: `keep_details` is never used, as we are not keeping details of
-    packages as this has no value.
-    """
-    packages = []
-
-    # Collect current data
-    current_packages = getattr(resource, 'packages') or []
-
-    if TRACE_LIGHT and current_packages:
-        from packagedcode.models import Package
-        packs = [Package.create(**p) for p in current_packages]
-        logger_debug('package_summarizer: for:', resource,
-                     'current_packages are:', packs)
-
-    current_packages = add_files(current_packages, resource)
-    packages.extend(current_packages)
-
-    if TRACE_LIGHT and packages:
-        logger_debug()
-        from packagedcode.models import Package  # NOQA
-        packs = [Package.create(**p) for p in packages]
-        logger_debug('package_summarizer: for:', resource,
-                     'packages are:', packs)
-
-    # Collect direct children packages summary
-    for child in children:
-        child_summaries = get_resource_summary(child, key='packages', as_attribute=False) or []
-        packages.extend(child_summaries)
-
-    # summarize proper
-    set_resource_summary(resource, key='packages', value=packages, as_attribute=False)
-    return packages
+        pruned_tallies.append(t)
+    return pruned_tallies
 
 
-def get_declared_holder(codebase, holders_summary):
+def get_declared_holder(codebase, holders_tallies):
     """
     Determine the declared holders of a codebase from the holders detected from
     key files.
@@ -338,7 +146,7 @@ def get_declared_holder(codebase, holders_summary):
     """
     entry_by_holders = {
         fingerprints.generate(entry['value']): entry
-        for entry in holders_summary if entry['value']
+        for entry in holders_tallies if entry['value']
     }
     key_file_holders = get_field_values_from_codebase_resources(codebase, 'holders', key_files_only=True)
     entry_by_key_file_holders = {
@@ -367,13 +175,13 @@ def get_declared_holder(codebase, holders_summary):
     return declared_holder
 
 
-def get_primary_language(programming_language_summary):
+def get_primary_language(programming_language_tallies):
     """
     Return the most common detected programming language as the primary language.
     """
     programming_languages_by_count = {
         entry['count']: entry['value']
-        for entry in programming_language_summary
+        for entry in programming_language_tallies
     }
     primary_language = ''
     if programming_languages_by_count: