Various v2.9.0 changes

CHANGELOG.rst

@@ -116,6 +116,7 @@ Changed
   executions. (improvement) #4328
 * Change default NGINX configuration to use HTTP 308 redirect, rather than 301, for plaintext requests.
   #4335
+* Improve performance of the ``GET /v1/actions/views/overview`` API endpoint. (improvement) #4337
 
 Fixed
 ~~~~~

contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 from __future__ import absolute_import
-import copy
 import eventlet
 import traceback
 import uuid
@@ -46,6 +45,7 @@
 from st2common.util import jinja as jinja_utils
 from st2common.util import param as param_utils
 from st2common.util.config_loader import get_config
+from st2common.util.ujson import fast_deepcopy
 
 __all__ = [
     'ActionChainRunner',
@@ -87,7 +87,7 @@ def init_vars(self, action_parameters):
                                                 action_parameters=action_parameters)
 
     def restore_vars(self, ctx_vars):
-        self.vars.update(copy.deepcopy(ctx_vars))
+        self.vars.update(fast_deepcopy(ctx_vars))
 
     def validate(self):
         """

contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py

@@ -15,7 +15,6 @@
 
 from __future__ import absolute_import
 
-import copy
 import uuid
 
 from oslo_config import cfg
@@ -31,7 +30,7 @@
 from st2common.services import action as ac_svc
 from st2common.services import workflows as wf_svc
 from st2common.util import api as api_util
-
+from st2common.util import ujson
 
 __all__ = [
     'OrquestaRunner',
@@ -51,7 +50,7 @@ def get_workflow_definition(entry_point):
             return def_file.read()
 
     def _construct_context(self, wf_ex):
-        ctx = copy.deepcopy(self.context)
+        ctx = ujson.fast_deepcopy(self.context)
         ctx['workflow_execution'] = str(wf_ex.id)
 
         return ctx

st2api/st2api/app.py

@@ -77,8 +77,8 @@ def setup_app(config={}):
     app = ErrorHandlingMiddleware(app)
     app = CorsMiddleware(app)
     app = LoggingMiddleware(app, router)
-    app = ResponseInstrumentationMiddleware(app, service_name='api')
+    app = ResponseInstrumentationMiddleware(app, router, service_name='api')
     app = RequestIDMiddleware(app)
-    app = RequestInstrumentationMiddleware(app, service_name='api')
+    app = RequestInstrumentationMiddleware(app, router, service_name='api')
 
     return app

st2api/st2api/controllers/v1/action_views.py

@@ -88,7 +88,6 @@ def _get_one(action_id, requester_user):
                 GET /actions/views/parameters/1
         """
         action_db = LookupUtils._get_action_by_id(action_id)
-        LOG.info('Found action: %s, runner: %s', action_db, action_db.runner_type['name'])
 
         permission_type = PermissionType.ACTION_VIEW
         rbac_utils.assert_user_has_resource_db_permission(user_db=requester_user,
@@ -111,6 +110,13 @@ class OverviewController(resource.ContentPackResourceController):
         'sort': ['pack', 'name']
     }
 
+    mandatory_include_fields_retrieve = [
+        'pack',
+        'name',
+        'parameters',
+        'runner_type'
+    ]
+
     def get_one(self, ref_or_id, requester_user):
         """
             List action by id.
@@ -141,11 +147,37 @@ def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, o
                                                         limit=limit,
                                                         raw_filters=raw_filters,
                                                         requester_user=requester_user)
+        runner_type_names = set([])
+        action_ids = []
+
         result = []
         for item in resp.json:
             action_api = ActionAPI(**item)
-            result.append(self._transform_action_api(action_api=action_api,
-                                                     requester_user=requester_user))
+            result.append(action_api)
+
+            runner_type_names.add(action_api.runner_type)
+            action_ids.append(str(action_api.id))
+
+        # Add combined runner and action parameters to the compound result object
+        # NOTE: This approach results in 2 additional queries while previous one resulted in
+        # N * 2 additional queries
+
+        # 1. Retrieve all the respective runner objects - we only need parameters
+        runner_type_dbs = RunnerType.query(name__in=runner_type_names,
+                                           only_fields=['name', 'runner_parameters'])
+        runner_type_dbs = dict([(runner_db.name, runner_db) for runner_db in runner_type_dbs])
+
+        # 2. Retrieve all the respective action objects - we only need parameters
+        action_dbs = dict([(action_db.id, action_db) for action_db in result])
+
+        for action_api in result:
+            action_db = action_dbs.get(action_api.id, None)
+            runner_db = runner_type_dbs.get(action_api.runner_type, None)
+            all_params = action_param_utils.get_params_view(action_db=action_db,
+                                                            runner_db=runner_db,
+                                                            merged_only=True)
+            action_api.parameters = all_params
+
         resp.json = result
         return resp
 

st2api/st2api/controllers/v1/actionexecutions.py

@@ -86,7 +86,8 @@ class ActionExecutionsControllerMixin(BaseRestControllerMixin):
     # parameters
     mandatory_include_fields_retrieve = [
         'action.parameters',
-        'runner.runner_parameters'
+        'runner.runner_parameters',
+        'parameters'
     ]
 
     # A list of attributes which can be specified using ?exclude_attributes filter

st2auth/st2auth/app.py

@@ -71,8 +71,8 @@ def setup_app(config={}):
     app = ErrorHandlingMiddleware(app)
     app = CorsMiddleware(app)
     app = LoggingMiddleware(app, router)
-    app = ResponseInstrumentationMiddleware(app, service_name='auth')
+    app = ResponseInstrumentationMiddleware(app, router, service_name='auth')
     app = RequestIDMiddleware(app)
-    app = RequestInstrumentationMiddleware(app, service_name='auth')
+    app = RequestInstrumentationMiddleware(app, router, service_name='auth')
 
     return app

st2common/st2common/middleware/instrumentation.py

@@ -13,34 +13,52 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__all__ = [
-    'RequestInstrumentationMiddleware',
-    'ResponseInstrumentationMiddleware'
-]
-
 from webob import Request
 
+from st2common import log as logging
 from st2common.metrics.base import CounterWithTimer
 from st2common.metrics.base import get_driver
 from st2common.util.date import get_datetime_utc_now
+from st2common.router import NotFoundException
+
+__all__ = [
+    'RequestInstrumentationMiddleware',
+    'ResponseInstrumentationMiddleware'
+]
+
+LOG = logging.getLogger(__name__)
 
 
 class RequestInstrumentationMiddleware(object):
     """
     Instrumentation middleware which records various request related metrics.
     """
 
-    def __init__(self, app, service_name):
+    def __init__(self, app, router, service_name):
         """
         :param service_name: Service name (e.g. api, stream, auth).
         :type service_name: ``str``
         """
         self.app = app
+        self.router = router
         self._service_name = service_name
 
     def __call__(self, environ, start_response):
         request = Request(environ)
 
+        try:
+            endpoint, _ = self.router.match(request)
+        except NotFoundException:
+            endpoint = {}
+
+        # NOTE: We don't track per request and response metrics for /v1/executions/<id> and some
+        # other endpoints because this would result in a lot of unique metrics which is an
+        # anti-pattern and causes unnecessary load on the metrics server.
+        submit_metrics = endpoint.get('x-submit-metrics', True)
+        if not submit_metrics:
+            LOG.debug('Not submitting request metrics for path: %s' % (request.path))
+            return self.app(environ, start_response)
+
         metrics_driver = get_driver()
 
         key = '%s.request.total' % (self._service_name)
@@ -99,12 +117,13 @@ class ResponseInstrumentationMiddleware(object):
     Instrumentation middleware which records various response related metrics.
     """
 
-    def __init__(self, app, service_name):
+    def __init__(self, app, router, service_name):
         """
         :param service_name: Service name (e.g. api, stream, auth).
         :type service_name: ``str``
         """
         self.app = app
+        self.router = router
         self._service_name = service_name
 
     def __call__(self, environ, start_response):

st2common/st2common/models/db/execution.py

@@ -110,7 +110,7 @@ def mask_secrets(self, value):
         parameters.update(value.get('runner', {}).get('runner_parameters', {}))
 
         secret_parameters = get_secret_parameters(parameters=parameters)
-        result['parameters'] = mask_secret_parameters(parameters=result['parameters'],
+        result['parameters'] = mask_secret_parameters(parameters=result.get('parameters', {}),
                                                       secret_parameters=secret_parameters)
 
         if 'parameters' in liveaction:

st2common/st2common/models/utils/action_param_utils.py

@@ -14,12 +14,13 @@
 # limitations under the License.
 
 from __future__ import absolute_import
-import copy
+
 import six
 
 from st2common import log as logging
 from st2common.util import action_db as action_db_util
 from st2common.util.casts import get_cast
+from st2common.util.ujson import fast_deepcopy
 
 LOG = logging.getLogger(__name__)
 
@@ -49,8 +50,15 @@ def _merge_param_meta_values(action_meta=None, runner_meta=None):
 
 
 def get_params_view(action_db=None, runner_db=None, merged_only=False):
-    runner_params = copy.deepcopy(runner_db.runner_parameters) if runner_db else {}
-    action_params = copy.deepcopy(action_db.parameters) if action_db else {}
+    if runner_db:
+        runner_params = fast_deepcopy(getattr(runner_db, 'runner_parameters', {})) or {}
+    else:
+        runner_params = {}
+
+    if action_db:
+        action_params = fast_deepcopy(getattr(action_db, 'parameters', {})) or {}
+    else:
+        action_params = {}
 
     parameters = set(runner_params.keys()).union(set(action_params.keys()))
 

st2common/st2common/openapi.yaml

@@ -250,6 +250,7 @@ paths:
       operationId: st2api.controllers.v1.actions:actions_controller.get_all
       x-permissions: action_list
       description: Returns a list of all actions.
+      x-log-result: false
       parameters:
         - name: exclude_attributes
           in: query
@@ -460,6 +461,7 @@ paths:
       operationId: st2api.controllers.v1.action_views:overview_controller.get_all
       x-permissions: action_list
       description: Returns a list of all the actions with runner parameters included.
+      x-log-result: false
       parameters:
         - name: exclude_attributes
           in: query
@@ -1066,6 +1068,7 @@ paths:
     get:
       operationId: st2api.controllers.v1.actionexecutions:action_executions_controller.get_one
       x-log-result: false
+      x-submit-metrics: false
       description: |
         Get one execution.
       parameters:
@@ -1162,6 +1165,7 @@ paths:
     get:
       operationId: st2api.controllers.v1.actionexecutions:action_execution_output_controller.get_one
       x-log-result: false
+      x-submit-metrics: false
       description: |
         Retrieve execution output.
       parameters:
@@ -1195,6 +1199,7 @@ paths:
     post:
       operationId: st2api.controllers.v1.actionexecutions:action_execution_rerun_controller.post
       x-log-result: false
+      x-submit-metrics: false
       description: |
         Create a new execution.
       parameters:
@@ -1250,6 +1255,7 @@ paths:
     get:
       operationId: st2api.controllers.v1.actionexecutions:action_execution_attribute_controller.get
       x-log-result: false
+      x-submit-metrics: false
       description: |
         Get one execution.
       parameters:
@@ -1284,6 +1290,7 @@ paths:
     get:
       operationId: st2api.controllers.v1.actionexecutions:action_execution_children_controller.get_one
       x-log-result: false
+      x-submit-metrics: false
       description: |
         Get one execution.
       parameters:
@@ -1306,6 +1313,7 @@ paths:
           in: context
           x-as: requester_user
           description: User performing the operation.
+      x-submit-metrics: false
       responses:
         '200':
           description: Execution attribute requested
@@ -1900,6 +1908,7 @@ paths:
       x-requirements:
         file_path: .*
       x-log-result: false
+      x-submit-metrics: false
       description: Get information about an installed pack.
       parameters:
         - name: ref_or_id