diff --git a/schema/OffRegScraper.py b/schema/OffRegScraper.py new file mode 100644 index 0000000..1107d30 --- /dev/null +++ b/schema/OffRegScraper.py @@ -0,0 +1,30 @@ +from bs4 import BeautifulSoup +import requests +import re +import csv + +results = {} + +URL = "https://product-security.pages.redhat.com/offering-registry/" +r = requests.get(URL) + +soup = BeautifulSoup(r.text, 'html.parser') +table = soup.find("table") +rows = table.findAll("tr") + +for row in rows: + for elem in row.contents: + if row.contents[1].text == 'Offering': + break + else: + # We extract the short name of the URL + re_search = re.search('/offering-registry/offerings/(.*)/', row.contents[1].contents[0].attrs["href"]) + results[re_search.group(1)] = row.contents[1].contents[0].text + break + +print(results) + +with open('offerings.csv', 'w') as csv_file: + writer = csv.writer(csv_file) + for key, value in results.items(): + writer.writerow([key, value]) diff --git a/schema/populate.sql b/schema/populate.sql new file mode 100644 index 0000000..949b780 --- /dev/null +++ b/schema/populate.sql @@ -0,0 +1,126 @@ +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-automation-platform','Ansible Automation Platform (AAP)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('advisor','Insights Advisor'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-on-aws','Ansible on AWS'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-on-azure','Ansible on Azure'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-on-gcp','Ansible on GCP'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-wisdom-service','Ansible Wisdom Service'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('cert-manager','cert-manager Operator for Red Hat OpenShift'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('compliance','Insights Compliance'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('connected-customer-experience','Connected Customer Experience (CCX)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('cost-management','Cost Management'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('custom-metric-autoscaler','OpenShift Custom Metrics Autoscaler'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('developer-sandbox-for-red-hat-openshift','Developer Sandbox for Red Hat OpenShift'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('dotnet','.NET'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('drift','Insights Drift'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('eclipse-vertx','Red Hat build of Eclipse Vert.x'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('edge-management','Edge Management'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('eventing','Insights Eventing'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('fastdatapath','RHEL Fast Datapath'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('host-management-services','Host Management Services'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('hosted-control-planes','Hosted Control Planes (Hypershift)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('hybrid-application-console','Hybrid Application Console (HAC)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('insights-essential','Insights Essentials'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('kernel-module-management','Kernel Module Management'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('logging-subsystem-for-red-hat-openshift','Logging Subsystem for Red Hat OpenShift'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('lvms-operator','LVMS Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('malware-detection','Insights Malware Detection'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('mgmt-platform','Management Platform'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-applications','Migration Toolkit for Applications (MTA)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-containers','Migration Toolkit for Containers (MTC)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-runtimes','Migration Toolkit for Runtimes (MTR)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-virtualization','Migration Toolkit for Virtualization (MTV)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('network-observability-operator','Network Observability Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('node-healthcheck-operator','Node HealthCheck Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('node-maintenance-operator','Node Maintenance Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('nvidia-gpu-add-on','NVIDIA GPU Add-On'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('oadp','OpenShift API for Data Protection'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-container-platform','Openshift Container Platform (OCP)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-container-storage','OpenShift Container Storage (OCS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-data-foundation-managed-service','Red Hat OpenShift Data Foundation Managed Service'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-dedicated','OpenShift Dedicated (OSD/ROSA)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-developer-tools-and-services-helm','OpenShift Developer Tools and Services (Helm)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-developer-tools-and-services-jenkins','OpenShift Developer Tools and Services (Jenkins)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-distributed-tracing','OpenShift Distributed Tracing'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-on-azure','Openshift on Azure (ARO)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-run-once-duration-override-operator','OpenShift Run Once Duration Override Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-sandboxed-containers','Openshift Sandboxed Containers'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-secondary-scheduler-operator','OpenShift Secondary Scheduler Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-servicemesh','OpenShift Service Mesh'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-virtualization','OpenShift Virtualization (CNV)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-web-terminal-operator','OpenShift Web Terminal Operator'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-winc','Windows Container Support for OpenShift'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('patch','Insights Patch'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('product-discovery','Product Discovery'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-3scale-api-management-platform','Red Hat 3scale API Management Platform'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-advanced-cluster-management','Red Hat Advanced Cluster Management (RHACM)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-broker','Red Hat AMQ Broker'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-clients','Red Hat AMQ Clients'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-interconnect','Red Hat AMQ Interconnect'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-online','Red Hat AMQ Online'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-streams','Red Hat AMQ Streams'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-build-apicurio-registry','Red Hat build of Apicurio Registry (formerly known as Integration Service Registry)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-build-quarkus','Red Hat Build of Quarkus'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-camel-extensions-quarkus','Red Hat Camel Extensions for Quarkus'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-camel-k','Red Hat Camel K'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-camel-spring-boot','Red Hat Camel for Spring Boot'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-ceph-storage','Red Hat Ceph Storage'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-certificate-system','Red Hat Certificate System (RHCS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-certification-program','Red Hat Certification Program (rhcertification)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-code-quarkus','Red Hat Code Quarkus'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-core-os','Red Hat CoreOS'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-data-grid','Red Hat Data Grid'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-debezium','Red Hat Debezium'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-decision-manager','Red Hat Decision Manager'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-developer-hub','Red Hat Developer Hub'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-developer-toolset','Red Hat Developer Toolset (DTS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-devtools-compilers','Red Hat Developer Tools (DevTools Compilers)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-directory-server','Red Hat Directory Server (RHDS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-10','Red Hat Enterprise Linux (RHEL) 10'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-6','Red Hat Enterprise Linux (RHEL) 6'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-7','Red Hat Enterprise Linux (RHEL) 7'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-8','Red Hat Enterprise Linux (RHEL) 8'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-9','Red Hat Enterprise Linux (RHEL) 9'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-fuse','Red Hat Fuse'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-gluster-storage','Red Hat Gluster Storage'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-in-vehicle-os','Red Hat In-Vehicle Operating System (RHIVOS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-jboss-core-services','Red Hat JBoss Core Services'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-jboss-eap','Red Hat JBoss Enterprise Application Platform (EAP)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-jboss-web-server','Red Hat JBoss Web Server'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-observability-service','Red Hat Observability Service'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-open-database-access','Red Hat OpenShift Database Access'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-open-shift-data-science','Red Hat OpenShift Data Science (RHODS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openjdk','Red Hat OpenJDK'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-api-management','Red Hat OpenShift API Management'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-builds-v2','Red Hat OpenShift Builds V2'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-connectors','Red Hat OpenShift Connectors (RHOC)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-control-plane-service','Red Hat OpenShift Control Plane Service'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-data-foundation','Red Hat OpenShift Data Foundation'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-dev-spaces','Red Hat OpenShift Dev Spaces'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-gitops','Red Hat OpenShift GitOps'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-local','Red Hat OpenShift Local'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-pipelines','Red Hat OpenShift Pipelines'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-serverless','Red Hat OpenShift Serverless'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-service-registry','Red Hat OpenShift Service Registry'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-streams-apache-kafka','Red Hat OpenShift Streams for Apache Kafka (RHOSAK)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openstack-platform','Red Hat OpenStack Platform (RHOSP)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-optaplanner','Red Hat Optaplanner'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-plug-ins-for-backstage','Red Hat Plug-ins for Backstage'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-process-automation-manager','Red Hat Process Automation Manager'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-quarkus-registry','Red Hat Quarkus Registry'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-quay','Red Hat Quay'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-satellite','Red Hat Satellite'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-service-interconnect','Red Hat Service Interconnect (formerly known as Application Interconnect)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-single-sign-on','Red Hat Single Sign-On (RHSSO)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-software-collections','Red Hat Software Collections'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-support-for-spring-boot','Red Hat support for Spring Boot'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-trusted-application-pipeline','Red Hat Trusted Application Pipeline (RHTAP)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-update-infrastructure','Red Hat Update Infrastructure (RHUI)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-virtualization','Red Hat Virtualization'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('resource-optimization','Insights Resource Optimization (ROS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('rh-vulnerability-for-ocp','Insights Vulnerability for OCP'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('rhacs','Red Hat Advanced Cluster Security for Kubernetes (RHACS)'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('self-node-remediation','Self Node Remediation'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('subscription-central','Subscription Central'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('subscription-watch','Subscription Watch'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('telco-sw-components','Telco SW Components'); +INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('vulnerability','Vulnerability'); diff --git a/schema/schema.sql b/schema/schema.sql index 942b981..298edbc 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -2,34 +2,52 @@ CREATE SCHEMA covscanrest; GRANT USAGE ON SCHEMA covscanrest TO postgres; -CREATE TABLE IF NOT EXISTS covscanrest.products( - productID VARCHAR(100), +CREATE TABLE IF NOT EXISTS covscanrest.offerings( + offeringId VARCHAR(100), description VARCHAR(200), - repository VARCHAR (150), - PRIMARY KEY (productID) + PRIMARY KEY (offeringId) +); + +CREATE TABLE IF NOT EXISTS covscanrest.results( + resultsId SERIAL, + datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL, + state BOOLEAN, + logs bytea, + task_reference VARCHAR(50), + PRIMARY KEY (resultsId) ); -CREATE TABLE IF NOT EXISTS covscanrest.scans ( +CREATE TABLE IF NOT EXISTS covscanrest.scans( scanID SERIAL, - productID VARCHAR(100), + offeringId VARCHAR(100), eventID VARCHAR(100) NOT NULL, isManagedService BOOLEAN NOT NULL, componentList VARCHAR(100), datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL, + owner VARCHAR(50) NOT NULL, + results SERIAL, + status VARCHAR (50) CONSTRAINT valid_status CHECK(status in ('PENDING', 'DELETED', 'COMPLETED', 'IN PROGRESS')), + last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL, PRIMARY KEY(scanID), - FOREIGN KEY (productID) REFERENCES covscanrest.products(productID) + FOREIGN KEY (offeringId) REFERENCES covscanrest.offerings(offeringId), + FOREIGN KEY (results) REFERENCES covscanrest.results(resultsId) ); -CREATE TABLE IF NOT EXISTS covscanrest.archive ( +CREATE TABLE IF NOT EXISTS covscanrest.archive( scanID SERIAL, - productID VARCHAR(100), + offeringId VARCHAR(100), eventID VARCHAR(100) NOT NULL, isManagedService BOOLEAN NOT NULL, componentList VARCHAR(100), datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL, + owner VARCHAR(50) NOT NULL, + results SERIAL, + status VARCHAR (50) CONSTRAINT valid_status CHECK(status in ('PENDING', 'DELETED', 'COMPLETED', 'IN PROGRESS')), + last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL, PRIMARY KEY(scanID), - FOREIGN KEY (productID) REFERENCES covscanrest.products(productID) + FOREIGN KEY (offeringId) REFERENCES covscanrest.offerings(offeringId), + FOREIGN KEY (results) REFERENCES covscanrest.results(resultsId) ); CREATE TABLE IF NOT EXISTS covscanrest.gitscans ( @@ -38,6 +56,8 @@ CREATE TABLE IF NOT EXISTS covscanrest.gitscans ( repository VARCHAR(150), reference VARCHAR(100), commitId VARCHAR(100), + -- SHA256 has a length of 256 bits, so 256 bits would represent 64 hex characters + hashsum VARCHAR(64), PRIMARY KEY(id) ); @@ -45,8 +65,6 @@ CREATE TABLE IF NOT EXISTS covscanrest.pncscans( id SERIAL, buildSystemType VARCHAR(80), buildId VARCHAR(100), - reference VARCHAR(100), - commitId VARCHAR(100), PRIMARY KEY(id) );