jonny
/
covscanrest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Modified schema, create scraper and populate file for offerings file

db_schema
jperezde 3 years ago
parent
f502e758c7
commit
f3c9338181
3 changed files with 186 additions and 12 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 30
      schema/OffRegScraper.py
  2. 126
      schema/populate.sql
  3. 42
      schema/schema.sql

30
schema/OffRegScraper.py
Unescape View File

@ -0,0 +1,30 @@
from bs4 import BeautifulSoup
import requests
import re
import csv
results = {}
URL = "https://product-security.pages.redhat.com/offering-registry/"
r = requests.get(URL)
soup = BeautifulSoup(r.text, 'html.parser')
table = soup.find("table")
rows = table.findAll("tr")
for row in rows:
for elem in row.contents:
if row.contents[1].text == 'Offering':
break
else:
# We extract the short name of the URL
re_search = re.search('/offering-registry/offerings/(.*)/', row.contents[1].contents[0].attrs["href"])
results[re_search.group(1)] = row.contents[1].contents[0].text
break
print(results)
with open('offerings.csv', 'w') as csv_file:
writer = csv.writer(csv_file)
for key, value in results.items():
writer.writerow([key, value])

126
schema/populate.sql
Unescape View File

@ -0,0 +1,126 @@
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-automation-platform','Ansible Automation Platform (AAP)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('advisor','Insights Advisor');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-on-aws','Ansible on AWS');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-on-azure','Ansible on Azure');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-on-gcp','Ansible on GCP');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('ansible-wisdom-service','Ansible Wisdom Service');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('cert-manager','cert-manager Operator for Red Hat OpenShift');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('compliance','Insights Compliance');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('connected-customer-experience','Connected Customer Experience (CCX)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('cost-management','Cost Management');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('custom-metric-autoscaler','OpenShift Custom Metrics Autoscaler');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('developer-sandbox-for-red-hat-openshift','Developer Sandbox for Red Hat OpenShift');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('dotnet','.NET');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('drift','Insights Drift');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('eclipse-vertx','Red Hat build of Eclipse Vert.x');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('edge-management','Edge Management');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('eventing','Insights Eventing');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('fastdatapath','RHEL Fast Datapath');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('host-management-services','Host Management Services');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('hosted-control-planes','Hosted Control Planes (Hypershift)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('hybrid-application-console','Hybrid Application Console (HAC)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('insights-essential','Insights Essentials');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('kernel-module-management','Kernel Module Management');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('logging-subsystem-for-red-hat-openshift','Logging Subsystem for Red Hat OpenShift');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('lvms-operator','LVMS Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('malware-detection','Insights Malware Detection');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('mgmt-platform','Management Platform');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-applications','Migration Toolkit for Applications (MTA)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-containers','Migration Toolkit for Containers (MTC)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-runtimes','Migration Toolkit for Runtimes (MTR)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('migration-toolkit-for-virtualization','Migration Toolkit for Virtualization (MTV)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('network-observability-operator','Network Observability Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('node-healthcheck-operator','Node HealthCheck Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('node-maintenance-operator','Node Maintenance Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('nvidia-gpu-add-on','NVIDIA GPU Add-On');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('oadp','OpenShift API for Data Protection');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-container-platform','Openshift Container Platform (OCP)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-container-storage','OpenShift Container Storage (OCS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-data-foundation-managed-service','Red Hat OpenShift Data Foundation Managed Service');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-dedicated','OpenShift Dedicated (OSD/ROSA)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-developer-tools-and-services-helm','OpenShift Developer Tools and Services (Helm)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-developer-tools-and-services-jenkins','OpenShift Developer Tools and Services (Jenkins)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-distributed-tracing','OpenShift Distributed Tracing');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-on-azure','Openshift on Azure (ARO)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-run-once-duration-override-operator','OpenShift Run Once Duration Override Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-sandboxed-containers','Openshift Sandboxed Containers');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-secondary-scheduler-operator','OpenShift Secondary Scheduler Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-servicemesh','OpenShift Service Mesh');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-virtualization','OpenShift Virtualization (CNV)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-web-terminal-operator','OpenShift Web Terminal Operator');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('openshift-winc','Windows Container Support for OpenShift');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('patch','Insights Patch');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('product-discovery','Product Discovery');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-3scale-api-management-platform','Red Hat 3scale API Management Platform');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-advanced-cluster-management','Red Hat Advanced Cluster Management (RHACM)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-broker','Red Hat AMQ Broker');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-clients','Red Hat AMQ Clients');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-interconnect','Red Hat AMQ Interconnect');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-online','Red Hat AMQ Online');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-amq-streams','Red Hat AMQ Streams');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-build-apicurio-registry','Red Hat build of Apicurio Registry (formerly known as Integration Service Registry)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-build-quarkus','Red Hat Build of Quarkus');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-camel-extensions-quarkus','Red Hat Camel Extensions for Quarkus');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-camel-k','Red Hat Camel K');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-camel-spring-boot','Red Hat Camel for Spring Boot');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-ceph-storage','Red Hat Ceph Storage');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-certificate-system','Red Hat Certificate System (RHCS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-certification-program','Red Hat Certification Program (rhcertification)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-code-quarkus','Red Hat Code Quarkus');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-core-os','Red Hat CoreOS');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-data-grid','Red Hat Data Grid');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-debezium','Red Hat Debezium');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-decision-manager','Red Hat Decision Manager');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-developer-hub','Red Hat Developer Hub');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-developer-toolset','Red Hat Developer Toolset (DTS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-devtools-compilers','Red Hat Developer Tools (DevTools Compilers)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-directory-server','Red Hat Directory Server (RHDS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-10','Red Hat Enterprise Linux (RHEL) 10');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-6','Red Hat Enterprise Linux (RHEL) 6');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-7','Red Hat Enterprise Linux (RHEL) 7');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-8','Red Hat Enterprise Linux (RHEL) 8');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-enterprise-linux-9','Red Hat Enterprise Linux (RHEL) 9');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-fuse','Red Hat Fuse');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-gluster-storage','Red Hat Gluster Storage');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-in-vehicle-os','Red Hat In-Vehicle Operating System (RHIVOS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-jboss-core-services','Red Hat JBoss Core Services');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-jboss-eap','Red Hat JBoss Enterprise Application Platform (EAP)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-jboss-web-server','Red Hat JBoss Web Server');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-observability-service','Red Hat Observability Service');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-open-database-access','Red Hat OpenShift Database Access');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-open-shift-data-science','Red Hat OpenShift Data Science (RHODS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openjdk','Red Hat OpenJDK');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-api-management','Red Hat OpenShift API Management');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-builds-v2','Red Hat OpenShift Builds V2');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-connectors','Red Hat OpenShift Connectors (RHOC)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-control-plane-service','Red Hat OpenShift Control Plane Service');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-data-foundation','Red Hat OpenShift Data Foundation');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-dev-spaces','Red Hat OpenShift Dev Spaces');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-gitops','Red Hat OpenShift GitOps');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-local','Red Hat OpenShift Local');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-pipelines','Red Hat OpenShift Pipelines');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-serverless','Red Hat OpenShift Serverless');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-service-registry','Red Hat OpenShift Service Registry');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openshift-streams-apache-kafka','Red Hat OpenShift Streams for Apache Kafka (RHOSAK)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-openstack-platform','Red Hat OpenStack Platform (RHOSP)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-optaplanner','Red Hat Optaplanner');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-plug-ins-for-backstage','Red Hat Plug-ins for Backstage');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-process-automation-manager','Red Hat Process Automation Manager');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-quarkus-registry','Red Hat Quarkus Registry');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-quay','Red Hat Quay');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-satellite','Red Hat Satellite');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-service-interconnect','Red Hat Service Interconnect (formerly known as Application Interconnect)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-single-sign-on','Red Hat Single Sign-On (RHSSO)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-software-collections','Red Hat Software Collections');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-support-for-spring-boot','Red Hat support for Spring Boot');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-trusted-application-pipeline','Red Hat Trusted Application Pipeline (RHTAP)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-update-infrastructure','Red Hat Update Infrastructure (RHUI)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('red-hat-virtualization','Red Hat Virtualization');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('resource-optimization','Insights Resource Optimization (ROS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('rh-vulnerability-for-ocp','Insights Vulnerability for OCP');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('rhacs','Red Hat Advanced Cluster Security for Kubernetes (RHACS)');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('self-node-remediation','Self Node Remediation');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('subscription-central','Subscription Central');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('subscription-watch','Subscription Watch');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('telco-sw-components','Telco SW Components');
INSERT INTO covscanrest.offerings(offeringId,description) VALUES ('vulnerability','Vulnerability');

42
schema/schema.sql
Unescape View File

@ -2,34 +2,52 @@ CREATE SCHEMA covscanrest;
GRANT USAGE ON SCHEMA covscanrest TO postgres;
CREATE TABLE IF NOT EXISTS covscanrest.products(
productID VARCHAR(100),
CREATE TABLE IF NOT EXISTS covscanrest.offerings(
offeringId VARCHAR(100),
description VARCHAR(200),
repository VARCHAR (150),
PRIMARY KEY (productID)
PRIMARY KEY (offeringId)
);
CREATE TABLE IF NOT EXISTS covscanrest.results(
resultsId SERIAL,
datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL,
state BOOLEAN,
logs bytea,
task_reference VARCHAR(50),
PRIMARY KEY (resultsId)
);
CREATE TABLE IF NOT EXISTS covscanrest.scans (
CREATE TABLE IF NOT EXISTS covscanrest.scans(
scanID SERIAL,
productID VARCHAR(100),
offeringId VARCHAR(100),
eventID VARCHAR(100) NOT NULL,
isManagedService BOOLEAN NOT NULL,
componentList VARCHAR(100),
datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL,
owner VARCHAR(50) NOT NULL,
results SERIAL,
status VARCHAR (50) CONSTRAINT valid_status CHECK(status in ('PENDING', 'DELETED', 'COMPLETED', 'IN PROGRESS')),
last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL,
PRIMARY KEY(scanID),
FOREIGN KEY (productID) REFERENCES covscanrest.products(productID)
FOREIGN KEY (offeringId) REFERENCES covscanrest.offerings(offeringId),
FOREIGN KEY (results) REFERENCES covscanrest.results(resultsId)
);
CREATE TABLE IF NOT EXISTS covscanrest.archive (
CREATE TABLE IF NOT EXISTS covscanrest.archive(
scanID SERIAL,
productID VARCHAR(100),
offeringId VARCHAR(100),
eventID VARCHAR(100) NOT NULL,
isManagedService BOOLEAN NOT NULL,
componentList VARCHAR(100),
datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL,
owner VARCHAR(50) NOT NULL,
results SERIAL,
status VARCHAR (50) CONSTRAINT valid_status CHECK(status in ('PENDING', 'DELETED', 'COMPLETED', 'IN PROGRESS')),
last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT (NOW() AT TIME ZONE 'utc') NOT NULL,
PRIMARY KEY(scanID),
FOREIGN KEY (productID) REFERENCES covscanrest.products(productID)
FOREIGN KEY (offeringId) REFERENCES covscanrest.offerings(offeringId),
FOREIGN KEY (results) REFERENCES covscanrest.results(resultsId)
);
CREATE TABLE IF NOT EXISTS covscanrest.gitscans (
@ -38,6 +56,8 @@ CREATE TABLE IF NOT EXISTS covscanrest.gitscans (
repository VARCHAR(150),
reference VARCHAR(100),
commitId VARCHAR(100),
-- SHA256 has a length of 256 bits, so 256 bits would represent 64 hex characters
hashsum VARCHAR(64),
PRIMARY KEY(id)
);
@ -45,8 +65,6 @@ CREATE TABLE IF NOT EXISTS covscanrest.pncscans(
id SERIAL,
buildSystemType VARCHAR(80),
buildId VARCHAR(100),
reference VARCHAR(100),
commitId VARCHAR(100),
PRIMARY KEY(id)
);

Write Preview
Loading…
Cancel
Save