Merge branch 'tekton-pod-optimization' into 'main'

Converted SCM scan pipeline run to task run with 1 pod instead of 3, and other minor fixes

See merge request pct-security/covscanrest!22

k8s/stage/osh-client-tekton/pipline/osh-client-from-source-pipeline.yaml

@@ -1,104 +0,0 @@
-#requires running `tkn hub install task "git-cli"` first 
-apiVersion: tekton.dev/v1beta1
-kind: Pipeline
-metadata:
-  name: osh-client-from-source
-spec:
-  description: This pipeline clones a repo, git archives it then sends it to covscan to be scanned with snyk
-  params:
-
-  - name: repo-url
-    description: The SCMURL
-    type: string
-
-  - name: revision
-    description: The revision or tag
-    type: string
-
-  - name: mock-build-params
-    description: The parameters to pass to covscan mock-build
-    type: string
-
-  - name: archive-name
-    description: The name of the git archive file
-    type: string
-    default: $(context.pipelineRun.uid).tar.gz
-
-  workspaces:
-    - name: sources
-      description: This workspace contains our cloned sources and is temporary
-    - name: source-tars
-      description: This workspace contains our source tar gzips for covscan and is semi-persistant
-    - name: ssl-ca-directory
-      description: Location of CA bundle for ssl verification with internal services
-
-
-  tasks:
-    - name: clone
-      taskRef:
-        name: git-clone
-      workspaces:
-        - name: output
-          workspace: sources
-          subPath: $(context.pipelineRun.name)
-        - name: ssl-ca-directory
-          workspace: ssl-ca-directory
-      params:
-        - name: url
-          value: $(params.repo-url)
-        - name: revision
-          value: $(params.revision)
-        - name: verbose
-          value: true
-
-    - name: archive
-      runAfter:
-        - clone
-      taskRef:
-        name: git-cli
-      workspaces:
-        - name: source
-          workspace: sources
-          subPath: $(context.pipelineRun.name)
-        - name: source-tars
-          workspace: source-tars
-          subPath: $(context.pipelineRun.name)
-      params:
-        - name: USER_HOME
-          value: /home/git
-        - name: archive-name
-          value: $(params.archive-name)
-        - name: GIT_SCRIPT
-          value: |
-            git config --global --add safe.directory /workspace/source
-            git archive --format=tar.gz HEAD -o /workspace/source-tars/$(params.archive-name)
-
-            #results:
-            #- name: archive-name
-            #description: The name of the tar.gz we created
-
-    - name: covscan
-      params:
-        - name: targz-file
-          value: $(params.archive-name)
-        - name: mock-build-params
-          value: $(params.mock-build-params)
-      runAfter:
-        - archive
-      taskRef:
-        name: osh-scan-task-from-source
-      workspaces:
-        - name: source-tars
-          workspace: source-tars
-          subPath: $(context.pipelineRun.name)
-  finally:
-    - name: cleanup-workspace
-      params:
-        - name: clear-dir
-          value: $(context.pipelineRun.name)
-      taskRef:
-        name: cleanup-workspace
-      workspaces:
-        - name: sources
-          workspace: sources
-          #Note we don't provide a subpath, this way we can contain the whole folder

k8s/stage/osh-client-tekton/task/osh-client-from-source-clearup-workspace.yaml

@@ -1,25 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  name: cleanup-workspace
-spec:
-  params:
-  - name: cleanup
-    type: string
-    default: true
-    description: Should we actually cleanup the sources dir
-  - name: clear-dir
-    type: string
- 
-  workspaces:
-    - name: sources
-      description: Where we checked out our sources
-
-  steps:
-    - name: perform-buildid-scan
-      image: registry.access.redhat.com/ubi9/ubi:9.2-696
-
-      script: |
-        #!/bin/bash
-        echo "Clearing up sources form $(params.clear-dir)"
-        rm -rv /workspace/sources/$(params.clear-dir)

k8s/stage/osh-client-tekton/task/osh-client-from-source.yaml

@@ -1,72 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  name: osh-scan-task-from-source
-spec:
-  stepTemplate:
-    env:
-      - name: "HOME"
-        value: "/tekton/home"
-  params:
-  - name: targz-file
-    type: string
-    default: "source.tar.gz"
-    description: The filename of the tar.gz we'll be uploading to covscan
-
-  - name: mock-build-params
-    type: string
-    description: Parameters pushed to mock build
-    default: "-p snyk-only-unstable --tarball-build-script=:"
-
-  volumes:
-    - name: osh-client-kerb-vol
-      secret:
-        defaultMode: 292
-        optional: false
-        secretName: kerberos-keytab-osh
-
-    - name: osh-client-kerb-config-vol
-      configMap:
-        name: kerberos-config-osh-client
-        items:
-          - key: linux-krb5.conf
-            path: linux-krb5.conf
-        defaultMode: 292
-        optional: false
-
-    - name: osh-client-config-vol
-      configMap:
-        name: osh-client-config
-        items:
-          - key: client.conf
-            path: client.conf
-        optional: false
-
-  workspaces:
-    - name: source-tars
-      description: source tar gzips are kept here
-
-  steps:
-    - name: perform-buildid-scan
-      image: quay.io/pct-security/osh-wrapper-client:latest
-      workingDir: /home/covscan
-      volumeMounts:
-        - name: osh-client-kerb-vol
-          mountPath: /kerberos
-          readOnly: true
-
-        - name: osh-client-config-vol
-          mountPath: /etc/osh/client.conf
-          readOnly: true
-          subPath: client.conf
-
-        - name: osh-client-kerb-config-vol
-          mountPath: /etc/krb5.conf
-          readOnly: true
-          subPath: linux-krb5.conf
-
-      script: |
-        #!/bin/bash
-        echo $(params.mock-build-params)
-        echo $(params.targz-file)
-        covscan mock-build $(params.mock-build-params) /workspace/source-tars/$(params.targz-file)

k8s/stage/osh-client-tekton/task/osh-client-git-cli-modified.yaml

@@ -1,146 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  annotations:
-    tekton.dev/categories: Git
-    tekton.dev/displayName: git cli
-    tekton.dev/pipelines.minVersion: 0.21.0
-    tekton.dev/platforms: linux/amd64,linux/s390x,linux/ppc64le
-    tekton.dev/tags: git
-  creationTimestamp: "2023-06-20T22:58:05Z"
-  generation: 2
-  labels:
-    app.kubernetes.io/version: "0.4"
-    hub.tekton.dev/catalog: tekton
-  name: git-cli
-  namespace: pct-security-tooling
-  resourceVersion: "3453559180"
-  uid: 95fc93dd-8780-41ab-9477-b698762dc1de
-spec:
-  description: |-
-    This task can be used to perform git operations.
-    Git command that needs to be run can be passed as a script to the task. This task needs authentication to git in order to push after the git operation.
-  params:
-  - default: cgr.dev/chainguard/git:root-2.39@sha256:7759f87050dd8bacabe61354d75ccd7f864d6b6f8ec42697db7159eccd491139
-    description: |
-      The base image for the task.
-    name: BASE_IMAGE
-    type: string
-  - default: ""
-    description: |
-      Git user name for performing git operation.
-    name: GIT_USER_NAME
-    type: string
-  - default: ""
-    description: |
-      Git user email for performing git operation.
-    name: GIT_USER_EMAIL
-    type: string
-  - default: |
-      git help
-    description: The git script to run.
-    name: GIT_SCRIPT
-    type: string
-  - default: /root
-    description: |
-      Absolute path to the user's home directory. Set this explicitly if you are running the image as a non-root user or have overridden
-      the gitInitImage param with an image containing custom user configuration.
-    name: USER_HOME
-    type: string
-  - default: "true"
-    description: Log the commands that are executed during `git-clone`'s operation.
-    name: VERBOSE
-    type: string
-  results:
-  - description: The precise commit SHA after the git operation.
-    name: commit
-    type: string
-  - name: archive-name
-    type: string
-    description: The archive name produced by the git archive
-  steps:
-  - env:
-    - name: HOME
-      value: $(params.USER_HOME)
-    - name: PARAM_VERBOSE
-      value: $(params.VERBOSE)
-    - name: PARAM_USER_HOME
-      value: $(params.USER_HOME)
-    - name: WORKSPACE_OUTPUT_PATH
-      value: $(workspaces.output.path)
-    - name: WORKSPACE_SSH_DIRECTORY_BOUND
-      value: $(workspaces.ssh-directory.bound)
-    - name: WORKSPACE_SSH_DIRECTORY_PATH
-      value: $(workspaces.ssh-directory.path)
-    - name: WORKSPACE_BASIC_AUTH_DIRECTORY_BOUND
-      value: $(workspaces.basic-auth.bound)
-    - name: WORKSPACE_BASIC_AUTH_DIRECTORY_PATH
-      value: $(workspaces.basic-auth.path)
-    image: $(params.BASE_IMAGE)
-    name: git
-    resources: {}
-    script: |
-      #!/usr/bin/env sh
-      set -eu
-
-      if [ "${PARAM_VERBOSE}" = "true" ] ; then
-        set -x
-      fi
-
-      if [ "${WORKSPACE_BASIC_AUTH_DIRECTORY_BOUND}" = "true" ] ; then
-        cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" "${PARAM_USER_HOME}/.git-credentials"
-        cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.gitconfig" "${PARAM_USER_HOME}/.gitconfig"
-        chmod 400 "${PARAM_USER_HOME}/.git-credentials"
-        chmod 400 "${PARAM_USER_HOME}/.gitconfig"
-      fi
-
-      if [ "${WORKSPACE_SSH_DIRECTORY_BOUND}" = "true" ] ; then
-        cp -R "${WORKSPACE_SSH_DIRECTORY_PATH}" "${PARAM_USER_HOME}"/.ssh
-        chmod 700 "${PARAM_USER_HOME}"/.ssh
-        chmod -R 400 "${PARAM_USER_HOME}"/.ssh/*
-      fi
-
-      # Setting up the config for the git.
-      git config --global user.email "$(params.GIT_USER_EMAIL)"
-      git config --global user.name "$(params.GIT_USER_NAME)"
-
-      eval '$(params.GIT_SCRIPT)'
-
-      RESULT_SHA="$(git rev-parse HEAD | tr -d '\n')"
-      EXIT_CODE="$?"
-      if [ "$EXIT_CODE" != 0 ]
-      then
-        exit $EXIT_CODE
-      fi
-      # Make sure we don't add a trailing newline to the result!
-      printf "%s" "$RESULT_SHA" > "$(results.commit.path)"
-    workingDir: $(workspaces.source.path)
-  workspaces:
-  - description: custom source tar location
-    name: source-tars
-  - description: A workspace that contains the fetched git repository.
-    name: source
-  - description: |
-      An optional workspace that contains the files that need to be added to git. You can
-      access the workspace from your script using `$(workspaces.input.path)`, for instance:
-
-        cp $(workspaces.input.path)/file_that_i_want .
-        git add file_that_i_want
-        # etc
-    name: input
-    optional: true
-  - description: |
-      A .ssh directory with private key, known_hosts, config, etc. Copied to
-      the user's home before git commands are executed. Used to authenticate
-      with the git remote when performing the clone. Binding a Secret to this
-      Workspace is strongly recommended over other volume types.
-    name: ssh-directory
-    optional: true
-  - description: |
-      A Workspace containing a .gitconfig and .git-credentials file. These
-      will be copied to the user's home before any git commands are run. Any
-      other files in this Workspace are ignored. It is strongly recommended
-      to use ssh-directory over basic-auth whenever possible and to bind a
-      Secret to this Workspace over other volume types.
-    name: basic-auth
-    optional: true

k8s/stage/osh-client-tekton/task/osh-scan-scm-task.yaml

@@ -0,0 +1,104 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: osh-scan-scm-task
+spec:
+  stepTemplate:
+    env:
+      - name: "HOME"
+        value: "/tekton/home"
+  
+  params:
+  - name: repo-url
+    type: string
+    description: The SCMURL
+
+  - name: revision
+    type: string
+    description: The revision or tag
+
+  - name: mock-build-params
+    type: string
+    description: Parameters pushed to mock build
+    default: "snyk-only-unstable --tarball-build-script=:"
+
+  - name: archive-name
+    type: string
+    description: The name of the git archive file
+    default: $(context.taskRun.uid).tar.gz
+
+  - name: tarball-storage-dir
+    type: string
+    description: What directory the scan tar gz will be put into
+    default: /workspace/source-tars/$(context.taskRun.name)
+
+  - name: working-dir
+    type: string
+    description: Working directory for the task
+    default: /home/covscan
+
+  workspaces:
+    - name: source-tars
+      description: This workspace contains our source tar gzips for covscan and is semi-persistant
+    - name: ssl-ca-directory
+      description: Location of CA bundle for ssl verification with internal services
+
+
+  volumes:
+    - name: osh-client-kerb-vol
+      secret:
+        defaultMode: 384
+        optional: false
+        secretName: kerberos-keytab-osh
+
+    - name: osh-client-kerb-config-vol
+      configMap:
+        name: kerberos-config-osh-client
+        items:
+          - key: linux-krb5.conf
+            path: linux-krb5.conf
+        defaultMode: 384
+        optional: false
+
+    - name: osh-client-config-vol
+      configMap:
+        name: osh-client-config
+        items:
+          - key: client.conf
+            path: client.conf
+        optional: false
+
+  steps:
+    - name: perform-osh-scm-scan
+      image: quay.io/pct-security/osh-wrapper-client:latest
+      workingDir: $(params.working-dir)
+
+      volumeMounts:
+        - name: osh-client-kerb-vol
+          mountPath: /kerberos
+          readOnly: true
+
+        - name: osh-client-config-vol
+          mountPath: /etc/osh/client.conf
+          readOnly: true
+          subPath: client.conf
+
+        - name: osh-client-kerb-config-vol
+          mountPath: /etc/krb5.conf
+          readOnly: true
+          subPath: linux-krb5.conf
+
+      script: |
+        #!/bin/bash
+        set -x
+        echo $(params.working-dir)
+        echo $(params.repo-url)
+        echo $(params.revision)
+        echo $(params.mock-build-params)
+        echo $(params.archive-name)
+        git clone -v $(params.repo-url) -b $(params.revision)
+        git --git-dir=$(basename $(params.repo-url) .git)/.git archive --format=tar.gz HEAD -o $(params.working-dir)/$(params.archive-name)
+        mkdir $(params.tarball-storage-dir)
+        cp $(params.working-dir)/$(params.archive-name) $(params.tarball-storage-dir)/
+        
+        covscan mock-build $(params.mock-build-params) $(params.working-dir)/$(params.archive-name)

k8s/stage/osh-client-tekton/task/osh-scan-task.yaml

@@ -58,6 +58,7 @@ spec:
 
       script: |
         #!/bin/bash
+        set -x
         echo $(params.buildId)
         echo $(params.scanProfile)
-        covscan mock-build -p $(params.scanProfile) --brew-build $(params.buildId)
+        covscan mock-build $(params.scanProfile) --brew-build $(params.buildId)

src/main/java/com/redhat/pctsec/model/ScanRequest.java

@@ -68,7 +68,7 @@ public class ScanRequest {
     }
 
     @Column(name="scan_properties")
-    public String scanProperties;
+    public String scanProperties = "";
     public ScanRequest() {
 
     }

src/main/java/com/redhat/pctsec/model/ScanRequests.java

@@ -134,6 +134,6 @@ public class ScanRequests {
     public void propagateOptions(){
         //In future lets export this scan object as YAML
         //If its empy overwrite with the global options
-        getScanRequests().stream().filter(eso -> eso.getScanProperties() == null).forEach(sr -> sr.setScanProperties(getGlobalScanProperties()));
+        getScanRequests().stream().filter(eso -> eso.getScanProperties().equals("")).filter(eso -> eso.getType() != RequestType.BREW).forEach(sr -> sr.setScanProperties(getGlobalScanProperties()));
     }
 }

src/main/java/com/redhat/pctsec/tekton/TaskHandler.java

@@ -1,14 +1,8 @@
 package com.redhat.pctsec.tekton;
-import com.redhat.pctsec.model.RequestType;
 import com.redhat.pctsec.model.ScanTask;
 import com.redhat.pctsec.model.ScanTaskState;
 import io.fabric8.kubernetes.api.model.ConfigMapVolumeSource;
 import io.fabric8.kubernetes.api.model.PersistentVolumeClaimVolumeSource;
-import io.fabric8.kubernetes.api.model.PodSecurityContext;
-import io.fabric8.kubernetes.api.model.PodSecurityContextBuilder;
-import io.fabric8.tekton.client.DefaultTektonClient;
-import io.fabric8.tekton.client.TektonClient;
-import io.fabric8.tekton.client.DefaultTektonClient;
 import io.fabric8.tekton.client.TektonClient;
 import io.fabric8.tekton.pipeline.v1beta1.*;
 
@@ -17,9 +11,10 @@ import io.quarkus.vertx.ConsumeEvent;
 import io.smallrye.common.annotation.Blocking;
 import jakarta.inject.Inject;
 
-import org.apache.commons.lang3.RandomStringUtils;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
+import org.apache.commons.compress.utils.FileNameUtils;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -27,13 +22,13 @@ public class TaskHandler {
 
     @ConfigProperty(name = "quarkus.openshift.namespace")
     String NAMESPACE;
-    @ConfigProperty(name = "tekton.pipeline.ref")
+    @ConfigProperty(name = "tekton.scmscan.task.ref")
-    String PIPELINE_REFERENCE;
+    String SCM_SCAN_REFERENCE;
     @ConfigProperty(name = "tekton.service-account")
     String SERVICE_ACCOUNT;
 
-    @ConfigProperty(name = "tekton.task.ref")
+    @ConfigProperty(name = "tekton.brewscan.task.ref")
-    String TASK_REFERENCE;
+    String BREW_SCAN_TASK_REFERENCE;
 
     @Inject
     TektonClient tektonClient;
@@ -46,7 +41,8 @@ public class TaskHandler {
         switch(scanTask.getScanRequest().getType())
         {
         case BREW:
-            scanTask.setTektonRunId(invokeScanTask(scanTask.getScanRequest().brewBuild.buildRef, ""));
+            scanTask.setTektonRunId(invokeScanTask(scanTask.getScanRequest().brewBuild.buildRef, 
+                        scanTask.getScanRequest().getScanProperties()));
             scanTask.setState(ScanTaskState.RUNNING);
             break;
 
@@ -68,72 +64,65 @@ public class TaskHandler {
 
     public String invokeScanTask(String buildId, String mockbuildArgs) {
         // String buildId = "xterm-366-8.el9";
-        String scanProfile = "snyk-only-unstable";
+        // String scanProfile = "snyk-only-unstable";
 
         // random taskrun name generating for now
-        TaskRun taskRun = new TaskRunBuilder().withNewMetadata().withName("osh-scan-taskrun-" + RandomStringUtils.randomAlphanumeric(8).toLowerCase())
+        TaskRun taskRun = new TaskRunBuilder()
+                .withNewMetadata()
+                .withGenerateName("osh-brew-scan-taskrun-")
                 .endMetadata()
                 .withNewSpec()
                 .withServiceAccountName(SERVICE_ACCOUNT)
                 .withNewTaskRef()
-                .withName(TASK_REFERENCE)
+                .withName(BREW_SCAN_TASK_REFERENCE)
                 .endTaskRef()
                 .withParams(
                         new Param("buildId", new ArrayOrString(buildId)),
-                        new Param("scanProfile", new ArrayOrString(scanProfile)))
+                        new Param("scanProfile", new ArrayOrString(mockbuildArgs)))
                 .endSpec()
                 .build();
 
-        tektonClient.v1beta1().taskRuns().inNamespace(NAMESPACE).resource(taskRun).create();
+        TaskRun createdTaskRun = tektonClient.v1beta1().taskRuns().inNamespace(NAMESPACE).resource(taskRun).create();
 
-        return taskRun.getMetadata().getName();
+        return createdTaskRun.getMetadata().getName();
     }
 
     public String invokeOshScmScanPipeline(String repo, String ref, String mockBuildArgs) {
 
-        PodSecurityContext securityContext = new PodSecurityContextBuilder()
-                .withRunAsNonRoot(true)
-                .withRunAsUser(65532L)
-                .build();
-
-        WorkspaceBinding sourcesWorkspaceBinding = new WorkspaceBindingBuilder()
-                .withName("sources")
-                .withPersistentVolumeClaim(new PersistentVolumeClaimVolumeSource("osh-client-sources", null))
-                .build();
-
         WorkspaceBinding sourceTarsWorkspaceBinding = new WorkspaceBindingBuilder()
-                .withName("source-tars")
+            .withName("source-tars")
-                .withPersistentVolumeClaim(new PersistentVolumeClaimVolumeSource("osh-client-source-tars", null))
+            .withPersistentVolumeClaim(new PersistentVolumeClaimVolumeSource("osh-client-source-tars", null))
-                .build();
+            .build();
 
         WorkspaceBinding sslCaDirectoryWorkspaceBinding = new WorkspaceBindingBuilder()
-                .withName("ssl-ca-directory")
+            .withName("ssl-ca-directory")
-                .withConfigMap(new ConfigMapVolumeSource(null, null, "config-trusted-cabundle", null))
+            .withConfigMap(new ConfigMapVolumeSource(null, null, "config-trusted-cabundle", null))
-                .build();
+            .build();
-
+        
         List<WorkspaceBinding> workspaceBindings = new ArrayList<>();
-        workspaceBindings.add(sourcesWorkspaceBinding);
         workspaceBindings.add(sourceTarsWorkspaceBinding);
         workspaceBindings.add(sslCaDirectoryWorkspaceBinding);
 
-        PipelineRun pipelineRun = new PipelineRunBuilder()
+        TaskRun taskRun = new TaskRunBuilder()
-                .withNewMetadata().withName("osh-scm-scan-" + RandomStringUtils.randomAlphanumeric(8).toLowerCase()).endMetadata()
+        .withNewMetadata()
-                .withNewSpec()
+        .withGenerateName("osh-scm-scan-taskrun-")
-                .withNewPodTemplate()
+        .endMetadata()
-                .withSecurityContext(securityContext)
+        .withNewSpec()
-                .endPodTemplate()
+        .withServiceAccountName(SERVICE_ACCOUNT)
-                .withServiceAccountName(SERVICE_ACCOUNT)
+        .withNewTaskRef()
-                .withNewPipelineRef().withName(PIPELINE_REFERENCE).endPipelineRef()
+        .withName(SCM_SCAN_REFERENCE)
-                .addNewParam().withName("repo-url").withNewValue(repo).endParam()
+        .endTaskRef()
-                .addNewParam().withName("revision").withNewValue(ref).endParam()
+        .addNewParam().withName("repo-url").withNewValue(repo).endParam()
-                .addNewParam().withName("mock-build-params").withNewValue(mockBuildArgs).endParam()
+        .addNewParam().withName("revision").withNewValue(ref).endParam()
-                .withWorkspaces(workspaceBindings)
+        .addNewParam().withName("mock-build-params").withNewValue(mockBuildArgs).endParam()
-                .endSpec()
+        .addNewParam().withName("archive-name").withNewValue(FileNameUtils.getBaseName(repo) + "-" + ref + ".tar.gz").endParam()
-                .build();
+        .withWorkspaces(workspaceBindings)
-
+        .endSpec()
-        tektonClient.v1beta1().pipelineRuns().inNamespace(NAMESPACE).resource(pipelineRun).create();
+        .build();
-
+
-        return pipelineRun.getMetadata().getName();
+        TaskRun createdTaskRun = tektonClient.v1beta1().taskRuns().inNamespace(NAMESPACE).resource(taskRun).create();
+
+        return createdTaskRun.getMetadata().getName();
     }
 
 

src/main/resources/application.properties

@@ -128,8 +128,8 @@ quarkus.arc.remove-unused-beans=false
 ##########################################
 #   Tekton Specifics (Used in app)       #
 ##########################################
-tekton.pipeline.ref=osh-client-from-source
+tekton.scmscan.task.ref=osh-scan-scm-task
-tekton.task.ref=osh-scan-task
+tekton.brewscan.task.ref=osh-scan-task
 tekton.service-account=${quarkus.openshift.service-account}
 
 ##########################################