Merge pull request #2538 from nexB/new-generated-code-clues

Add new generated code clues

Author: pombredanne

src/summarycode/generated.py

@@ -9,7 +9,6 @@
 
 from itertools import islice
 
-
 from commoncode.datautils import Boolean
 from commoncode.text import toascii
 from plugincode.scan import ScanPlugin
@@ -19,7 +18,7 @@
 import typecode.contenttype
 
 """
-Tag files as generated.
+Tag files as generated code based on conspicuous strings.
 """
 
 # Tracing flag
@@ -77,7 +76,7 @@ def generated_scanner(location, **kwargs):
     return dict(is_generated=is_generated)
 
 
-GENERATED_KEYWORDS = tuple(g.lower() for g in (
+GENERATED_KEYWORDS_LOWERED = tuple(g.lower() for g in (
     'generated by',
     'auto-generated',
     'automatically generated',
@@ -157,25 +156,39 @@ def generated_scanner(location, **kwargs):
     'function: static codebooks autogenerated by',
 
     # yarn lock
-    'THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.',
-    'This file has been automatically created by',
-    'Please do not edit this file, all changes will be lost',
+    'this is an autogenerated file. do not edit this file directly.',
+    'this file has been automatically created by',
+    'please do not edit this file, all changes will be lost',
 
-    # protoc
-    'Generated by the protocol buffer compiler.  DO NOT EDIT!',
+    # thrift
+    'this is an automatically-generated file.',
+    'it could get re-generated if the allow_idl_generation flag is on',
 
-))
+    # aws sdk ruby
+    'warning about generated code',
+    'this file is generated. see the contributing guide for more information',
+    'this file is generated',
 
+    # seen in Go files:
+    'code generated file. do not edit',
+    'code generated by running "go generate". do not edit.'
 
-max_lines = 150
+    # protoc
+    'generated by the protocol buffer compiler. do not edit!',
+    'generated by the protocol buffer compiler.',
+))
 
 
-def get_generated_code_hint(location, generated_keywords=GENERATED_KEYWORDS):
+def get_generated_code_hint(
+    location,
+    max_lines=150,
+    generated_keywords=GENERATED_KEYWORDS_LOWERED
+):
     """
     Return a line of extracted text from a file if that file is likely
     generated source code.
 
-    for each of the the first few lines of a source code file
+    for each of the first few lines of a source code file
       if generated keywords are found in the line as lowercase
          yield the line text as a 'potentially_ generated' annotation
     """
@@ -184,9 +197,8 @@ def get_generated_code_hint(location, generated_keywords=GENERATED_KEYWORDS):
         return
     with open(location, 'rb') as filein:
         for line in islice(filein, max_lines):
-            text = toascii(line.strip())
-            textl = text.lower()
-            if any(kw in textl for kw in generated_keywords):
+            text = toascii(line.strip()).lower()
+            if any(kw in text.lower() for kw in generated_keywords):
                 # yield only the first 100 chars..
                 yield text[:100]
 

tests/summarycode/test_generated.py

@@ -20,62 +20,62 @@ class TestGeneratedCode(FileBasedTesting):
 
     def test_basic(self):
         expected = [
-            '// This file was generated by the JavaTM Architecture '
-            'for XML Binding(JAXB) Reference Implementation',
-            '// Generated on: 2011.08.01 at 11:35:59 AM CEST'
+            '// This file was generated by the JavaTM Architecture '.lower() +
+            'for XML Binding(JAXB) Reference Implementation'.lower(),
+            '// Generated on: 2011.08.01 at 11:35:59 AM CEST'.lower()
         ]
         test_file = self.get_test_loc('generated/simple/generated_1.java')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_basic_alt(self):
         expected = [
-            '// This file was generated by the JavaTM Architecture '
-            'for XML Binding(JAXB) Reference Implementation',
-            '// Generated on: 2013.11.15 at 04:17:00 PM CET'
+            '// This file was generated by the JavaTM Architecture '.lower() +
+            'for XML Binding(JAXB) Reference Implementation'.lower(),
+            '// Generated on: 2013.11.15 at 04:17:00 PM CET'.lower()
         ]
         test_file = self.get_test_loc('generated/simple/generated_3.java')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_basic2(self):
-        expected = ['* This class was generated by the JAX-WS RI.']
+        expected = ['* This class was generated by the JAX-WS RI.'.lower()]
         test_file = self.get_test_loc('generated/simple/generated_2.java')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_basic3(self):
-        expected = ['/* This class was automatically generated']
+        expected = ['/* This class was automatically generated'.lower()]
         test_file = self.get_test_loc('generated/simple/generated_4.java')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_basic4(self):
         expected = [
-            '* <p>The following schema fragment specifies the '
-            'expected content contained within this class.'
+            '* <p>The following schema fragment specifies the '.lower() +
+            'expected content contained within this class.'.lower()
         ]
         test_file = self.get_test_loc('generated/simple/generated_5.java')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_basic5(self):
-        expected = ['/* DO NOT EDIT THIS FILE - it is machine generated */']
+        expected = ['/* DO NOT EDIT THIS FILE - it is machine generated */'.lower()]
         test_file = self.get_test_loc('generated/simple/generated_6.c')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_configure(self):
         expected = [
-            '# Generated by GNU Autoconf 2.64 for Apache CouchDB 1.0.1.'
+            '# Generated by GNU Autoconf 2.64 for Apache CouchDB 1.0.1.'.lower()
         ]
         test_file = self.get_test_loc('generated/simple/configure')
         result = list(generated.get_generated_code_hint(location=test_file))
         assert result == expected
 
     def test_tomcat_jspc(self):
         expected = [
-            '<!--Automatically created by Apache Jakarta Tomcat JspC.'
+            '<!--Automatically created by Apache Jakarta Tomcat JspC.'.lower()
         ]
         test_file = self.get_test_loc('generated/jspc/web.xml')
         result = list(generated.get_generated_code_hint(location=test_file))