Skip to content

Add new generated code clues #2538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pombredanne merged 3 commits into from
Jul 11, 2021
Merged

Add new generated code clues #2538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
98af887
Add new generated keywords
pombredanne Sep 4, 2020
e7bc24e
Add new misc. generated code clues
pombredanne Jun 4, 2021
adc8a66
Update expected results for generated code
pombredanne Jun 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 27 additions & 15 deletions src/summarycode/generated.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

from itertools import islice


from commoncode.datautils import Boolean
from commoncode.text import toascii
from plugincode.scan import ScanPlugin
Expand All @@ -19,7 +18,7 @@
import typecode.contenttype

"""
Tag files as generated.
Tag files as generated code based on conspicuous strings.
"""

# Tracing flag
Expand Down Expand Up @@ -77,7 +76,7 @@ def generated_scanner(location, **kwargs):
return dict(is_generated=is_generated)


GENERATED_KEYWORDS = tuple(g.lower() for g in (
GENERATED_KEYWORDS_LOWERED = tuple(g.lower() for g in (
'generated by',
'auto-generated',
'automatically generated',
Expand Down Expand Up @@ -157,25 +156,39 @@ def generated_scanner(location, **kwargs):
'function: static codebooks autogenerated by',

# yarn lock
'THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.',
'This file has been automatically created by',
'Please do not edit this file, all changes will be lost',
'this is an autogenerated file. do not edit this file directly.',
'this file has been automatically created by',
'please do not edit this file, all changes will be lost',

# protoc
'Generated by the protocol buffer compiler. DO NOT EDIT!',
# thrift
'this is an automatically-generated file.',
'it could get re-generated if the allow_idl_generation flag is on',

))
# aws sdk ruby
'warning about generated code',
'this file is generated. see the contributing guide for more information',
'this file is generated',

# seen in Go files:
'code generated file. do not edit',
'code generated by running "go generate". do not edit.'

max_lines = 150
# protoc
'generated by the protocol buffer compiler. do not edit!',
'generated by the protocol buffer compiler.',
))


def get_generated_code_hint(location, generated_keywords=GENERATED_KEYWORDS):
def get_generated_code_hint(
location,
max_lines=150,
generated_keywords=GENERATED_KEYWORDS_LOWERED
):
"""
Return a line of extracted text from a file if that file is likely
generated source code.

for each of the the first few lines of a source code file
for each of the first few lines of a source code file
if generated keywords are found in the line as lowercase
yield the line text as a 'potentially_ generated' annotation
"""
Expand All @@ -184,9 +197,8 @@ def get_generated_code_hint(location, generated_keywords=GENERATED_KEYWORDS):
return
with open(location, 'rb') as filein:
for line in islice(filein, max_lines):
text = toascii(line.strip())
textl = text.lower()
if any(kw in textl for kw in generated_keywords):
text = toascii(line.strip()).lower()
if any(kw in text.lower() for kw in generated_keywords):
# yield only the first 100 chars..
yield text[:100]

26 changes: 13 additions & 13 deletions tests/summarycode/test_generated.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,62 +20,62 @@ class TestGeneratedCode(FileBasedTesting):

def test_basic(self):
expected = [
'// This file was generated by the JavaTM Architecture '
'for XML Binding(JAXB) Reference Implementation',
'// Generated on: 2011.08.01 at 11:35:59 AM CEST'
'// This file was generated by the JavaTM Architecture '.lower() +
'for XML Binding(JAXB) Reference Implementation'.lower(),
'// Generated on: 2011.08.01 at 11:35:59 AM CEST'.lower()
]
test_file = self.get_test_loc('generated/simple/generated_1.java')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_basic_alt(self):
expected = [
'// This file was generated by the JavaTM Architecture '
'for XML Binding(JAXB) Reference Implementation',
'// Generated on: 2013.11.15 at 04:17:00 PM CET'
'// This file was generated by the JavaTM Architecture '.lower() +
'for XML Binding(JAXB) Reference Implementation'.lower(),
'// Generated on: 2013.11.15 at 04:17:00 PM CET'.lower()
]
test_file = self.get_test_loc('generated/simple/generated_3.java')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_basic2(self):
expected = ['* This class was generated by the JAX-WS RI.']
expected = ['* This class was generated by the JAX-WS RI.'.lower()]
test_file = self.get_test_loc('generated/simple/generated_2.java')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_basic3(self):
expected = ['/* This class was automatically generated']
expected = ['/* This class was automatically generated'.lower()]
test_file = self.get_test_loc('generated/simple/generated_4.java')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_basic4(self):
expected = [
'* <p>The following schema fragment specifies the '
'expected content contained within this class.'
'* <p>The following schema fragment specifies the '.lower() +
'expected content contained within this class.'.lower()
]
test_file = self.get_test_loc('generated/simple/generated_5.java')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_basic5(self):
expected = ['/* DO NOT EDIT THIS FILE - it is machine generated */']
expected = ['/* DO NOT EDIT THIS FILE - it is machine generated */'.lower()]
test_file = self.get_test_loc('generated/simple/generated_6.c')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_configure(self):
expected = [
'# Generated by GNU Autoconf 2.64 for Apache CouchDB 1.0.1.'
'# Generated by GNU Autoconf 2.64 for Apache CouchDB 1.0.1.'.lower()
]
test_file = self.get_test_loc('generated/simple/configure')
result = list(generated.get_generated_code_hint(location=test_file))
assert result == expected

def test_tomcat_jspc(self):
expected = [
'<!--Automatically created by Apache Jakarta Tomcat JspC.'
'<!--Automatically created by Apache Jakarta Tomcat JspC.'.lower()
]
test_file = self.get_test_loc('generated/jspc/web.xml')
result = list(generated.get_generated_code_hint(location=test_file))
Expand Down