From 98af8879e253362645d1d22633c5051766f3909e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Sep 2020 15:58:25 +0200 Subject: [PATCH 1/3] Add new generated keywords Signed-off-by: Philippe Ombredanne --- src/summarycode/generated.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/summarycode/generated.py b/src/summarycode/generated.py index c7c2da8b18d..36917764fe0 100644 --- a/src/summarycode/generated.py +++ b/src/summarycode/generated.py @@ -157,12 +157,24 @@ def generated_scanner(location, **kwargs): 'function: static codebooks autogenerated by', # yarn lock - 'THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.', - 'This file has been automatically created by', - 'Please do not edit this file, all changes will be lost', + 'this is an autogenerated file. do not edit this file directly.', + 'this file has been automatically created by', + 'please do not edit this file, all changes will be lost', + + # thrift + 'this is an automatically-generated file.', + 'it could get re-generated if the allow_idl_generation flag is on', + + # aws sdk ruby + 'warning about generated code', + 'this file is generated. see the contributing guide for more information', + 'this file is generated', + + # seen in Go files: + 'code generated file. do not edit', # protoc - 'Generated by the protocol buffer compiler. DO NOT EDIT!', + 'generated by the protocol buffer compiler. do not edit!', )) From e7bc24e4ea75195d22f032f5ab468da8c5d6ba2a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Jun 2021 17:18:47 +0200 Subject: [PATCH 2/3] Add new misc. generated code clues Signed-off-by: Philippe Ombredanne --- src/summarycode/generated.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/summarycode/generated.py b/src/summarycode/generated.py index 36917764fe0..b544acae958 100644 --- a/src/summarycode/generated.py +++ b/src/summarycode/generated.py @@ -9,7 +9,6 @@ from itertools import islice - from commoncode.datautils import Boolean from commoncode.text import toascii from plugincode.scan import ScanPlugin @@ -19,7 +18,7 @@ import typecode.contenttype """ -Tag files as generated. +Tag files as generated code based on conspicuous strings. """ # Tracing flag @@ -77,7 +76,7 @@ def generated_scanner(location, **kwargs): return dict(is_generated=is_generated) -GENERATED_KEYWORDS = tuple(g.lower() for g in ( +GENERATED_KEYWORDS_LOWERED = tuple(g.lower() for g in ( 'generated by', 'auto-generated', 'automatically generated', @@ -160,7 +159,7 @@ def generated_scanner(location, **kwargs): 'this is an autogenerated file. do not edit this file directly.', 'this file has been automatically created by', 'please do not edit this file, all changes will be lost', - + # thrift 'this is an automatically-generated file.', 'it could get re-generated if the allow_idl_generation flag is on', @@ -172,22 +171,24 @@ def generated_scanner(location, **kwargs): # seen in Go files: 'code generated file. do not edit', + 'code generated by running "go generate". do not edit.' # protoc - 'generated by the protocol buffer compiler. do not edit!', - + 'generated by the protocol buffer compiler. do not edit!', + 'generated by the protocol buffer compiler.', )) -max_lines = 150 - - -def get_generated_code_hint(location, generated_keywords=GENERATED_KEYWORDS): +def get_generated_code_hint( + location, + max_lines=150, + generated_keywords=GENERATED_KEYWORDS_LOWERED +): """ Return a line of extracted text from a file if that file is likely generated source code. - for each of the the first few lines of a source code file + for each of the first few lines of a source code file if generated keywords are found in the line as lowercase yield the line text as a 'potentially_ generated' annotation """ @@ -196,9 +197,8 @@ def get_generated_code_hint(location, generated_keywords=GENERATED_KEYWORDS): return with open(location, 'rb') as filein: for line in islice(filein, max_lines): - text = toascii(line.strip()) - textl = text.lower() - if any(kw in textl for kw in generated_keywords): + text = toascii(line.strip()).lower() + if any(kw in text.lower() for kw in generated_keywords): # yield only the first 100 chars.. yield text[:100] From adc8a66ad8695ee07c835c7a6fe0d52427d0b0d4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 17 Jun 2021 17:03:44 +0200 Subject: [PATCH 3/3] Update expected results for generated code Signed-off-by: Philippe Ombredanne --- tests/summarycode/test_generated.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/summarycode/test_generated.py b/tests/summarycode/test_generated.py index e7cc81a0957..84fd336605e 100644 --- a/tests/summarycode/test_generated.py +++ b/tests/summarycode/test_generated.py @@ -20,9 +20,9 @@ class TestGeneratedCode(FileBasedTesting): def test_basic(self): expected = [ - '// This file was generated by the JavaTM Architecture ' - 'for XML Binding(JAXB) Reference Implementation', - '// Generated on: 2011.08.01 at 11:35:59 AM CEST' + '// This file was generated by the JavaTM Architecture '.lower() + + 'for XML Binding(JAXB) Reference Implementation'.lower(), + '// Generated on: 2011.08.01 at 11:35:59 AM CEST'.lower() ] test_file = self.get_test_loc('generated/simple/generated_1.java') result = list(generated.get_generated_code_hint(location=test_file)) @@ -30,44 +30,44 @@ def test_basic(self): def test_basic_alt(self): expected = [ - '// This file was generated by the JavaTM Architecture ' - 'for XML Binding(JAXB) Reference Implementation', - '// Generated on: 2013.11.15 at 04:17:00 PM CET' + '// This file was generated by the JavaTM Architecture '.lower() + + 'for XML Binding(JAXB) Reference Implementation'.lower(), + '// Generated on: 2013.11.15 at 04:17:00 PM CET'.lower() ] test_file = self.get_test_loc('generated/simple/generated_3.java') result = list(generated.get_generated_code_hint(location=test_file)) assert result == expected def test_basic2(self): - expected = ['* This class was generated by the JAX-WS RI.'] + expected = ['* This class was generated by the JAX-WS RI.'.lower()] test_file = self.get_test_loc('generated/simple/generated_2.java') result = list(generated.get_generated_code_hint(location=test_file)) assert result == expected def test_basic3(self): - expected = ['/* This class was automatically generated'] + expected = ['/* This class was automatically generated'.lower()] test_file = self.get_test_loc('generated/simple/generated_4.java') result = list(generated.get_generated_code_hint(location=test_file)) assert result == expected def test_basic4(self): expected = [ - '*

The following schema fragment specifies the ' - 'expected content contained within this class.' + '*

The following schema fragment specifies the '.lower() + + 'expected content contained within this class.'.lower() ] test_file = self.get_test_loc('generated/simple/generated_5.java') result = list(generated.get_generated_code_hint(location=test_file)) assert result == expected def test_basic5(self): - expected = ['/* DO NOT EDIT THIS FILE - it is machine generated */'] + expected = ['/* DO NOT EDIT THIS FILE - it is machine generated */'.lower()] test_file = self.get_test_loc('generated/simple/generated_6.c') result = list(generated.get_generated_code_hint(location=test_file)) assert result == expected def test_configure(self): expected = [ - '# Generated by GNU Autoconf 2.64 for Apache CouchDB 1.0.1.' + '# Generated by GNU Autoconf 2.64 for Apache CouchDB 1.0.1.'.lower() ] test_file = self.get_test_loc('generated/simple/configure') result = list(generated.get_generated_code_hint(location=test_file)) @@ -75,7 +75,7 @@ def test_configure(self): def test_tomcat_jspc(self): expected = [ - '