Merge pull request #2694 from project-koku/rc-2021.03.01

Rc 2021.03.01

Author: cgoodfred

.env.example

@@ -13,13 +13,13 @@ DEVELOPMENT=True
 prometheus_multiproc_dir='/tmp'
 AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY
 AWS_SECRET_ACCESS_KEY=AWS_SECRET_KEY
-ENABLE_S3_ARCHIVING=False
-ENABLE_PARQUET_PROCESSING=False
+
+ENABLE_S3_ARCHIVING=True
+ENABLE_PARQUET_PROCESSING=True
 S3_BUCKET_NAME=koku-bucket
-S3_BUCKET_PATH=data_archive
-S3_ENDPOINT=s3.us-east-1.amazonaws.com
-S3_ACCESS_KEY=S3_ACCESS_KEY
-S3_SECRET=S3_SECRET
+S3_ENDPOINT=http://kokuminio:9000
+S3_ACCESS_KEY=kokuminioaccess
+S3_SECRET=kokuminiosecret
 
 # Local MinIO.  Defaults are set for docker-compose settings
 #S3_BUCKET_NAME=koku-bucket

.github/workflows/unittests.yml

@@ -33,8 +33,10 @@ jobs:
         ports:
           - 5432:5432
     steps:
-      - name: Checkout repsository
+      - name: Checkout repository
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Display build environment
         run: printenv

.gitignore

@@ -134,3 +134,6 @@ testing/
 # Local OpenShift folders
 openshift.local.clusterup/
 scripts/openshift.local.clusterup/
+
+# Localdev Certs
+certs/

Makefile

@@ -37,9 +37,9 @@ OC_TEMPLATES = $(wildcard $(OC_TEMPLATE_DIR))
 
 # Docker compose specific file
 ifdef compose_file
-    DOCKER_COMPOSE = docker-compose -f $(compose_file)
+    DOCKER_COMPOSE = $(DOCKER)-compose -f $(compose_file)
 else
-	DOCKER_COMPOSE = docker-compose
+	DOCKER_COMPOSE = $(DOCKER)-compose
 endif
 
 # Platform differences
@@ -115,6 +115,9 @@ help:
 	@echo "                                         password: admin12"
 	@echo "  docker-up-min                        run database, koku/masu servers and worker"
 	@echo "  docker-down                          shut down all containers"
+	@echo "  docker-up-min-presto                 start minimum targets for Presto usage"
+	@echo "  docker-up-min-presto-no-build        start minimum targets for Presto usage without building koku base"
+	@echo "  docker-presto-down-all               Tear down Presto and Koku containers"
 	@echo "  docker-rabbit                        run RabbitMQ container"
 	@echo "  docker-reinitdb                      drop and recreate the database"
 	@echo "  docker-reinitdb-with-sources         drop and recreate the database with fake sources"
@@ -168,7 +171,7 @@ create-test-customer-no-sources: run-migrations docker-up-koku
 	$(PYTHON) $(TOPDIR)/scripts/create_test_customer.py --no-sources --bypass-api || echo "WARNING: create_test_customer failed unexpectedly!"
 
 load-test-customer-data:
-	$(TOPDIR)/scripts/load_test_customer_data.sh $(TOPDIR) $(start) $(end)
+	$(TOPDIR)/scripts/load_test_customer_data.sh $(start) $(end)
 	make load-aws-org-unit-tree
 
 load-aws-org-unit-tree:
@@ -400,6 +403,8 @@ docker-presto-down:
 	docker-compose -f ./testing/compose_files/docker-compose-presto.yml down -v
 	make docker-presto-cleanup
 
+docker-presto-down-all: docker-presto-down docker-down
+
 ### Source targets ###
 ocp-source-from-yaml:
 #parameter validation

Pipfile.lock

@@ -173,6 +173,13 @@ If you see this error, run the following command (assuming you are at the projec
 
     setfacl -m u:26:-wx ./pg_data
 
+If you need to run PostgreSQL using SSL for local development:
+
+1. Ensure that you have ``openssl`` installed.
+2. Ensure that you have the ``KOKU_PATH`` environment set to the koku source root directory.
+3. Execute ``${KOKU_PATH}/scripts/genssc`` and follow the prompts to create local a self-signed certificate for PostgreSQL.
+4. In the ``docker-compose.yml`` file, there are two commented sections labeled "PostgreSQL SSL". Follow the directions in these comments to enable the volume mounts and command options to use SSL.
+
 See  https://access.redhat.com/containers/?tab=overview#/registry.access.redhat.com/rhel8/postgresql-12
 
 
@@ -217,6 +224,51 @@ Information about PostgreSQL statistics can be found here: https://www.postgresq
 Information about Grafana dashboards can be found here: https://grafana.com/docs/grafana/latest/features/dashboard/dashboards/
 
 
+Using Trino and MinIO
+^^^^^^^^^^^^^^^^^^^^^
+
+We have a special docker-compose file specifically for running Trino (formerly Presto) with MinIO for object storage. With the proper environment variables set the app will run circumventing our conventional Postgres processing in favor of using Trino.
+
+Set the following environment variables ::
+
+    ENABLE_PARQUET_PROCESSING=True
+    S3_BUCKET_NAME=koku-bucket
+    S3_ENDPOINT=http://kokuminio:9000
+    S3_ACCESS_KEY=kokuminioaccess
+    S3_SECRET=kokuminiosecret
+
+To spin up the minimum targets for Trino use ::
+
+    make docker-up-min-presto
+
+To skip building the koku image base ::
+
+    make docker-up-min-presto-no-build
+
+To tear down containers ::
+
+    make docker-presto-down-all
+
+With all containers running any source added will be processed by saving CSV files in MinIO and storing Parquet files in MinIO. The source's data will be summarized via Trino. Summarized data will land in the appropriate daily_summary table for the source type for consumption by the API.
+
+To add test sources and data ::
+
+    make create-test-customer
+    make load-test-customer-data (optional)start={start_date} (optional)end={end_date}
+
+The MinIO UI will be available at http://127.0.0.1:9000/minio/. Use the S3_ACCESS_KEY and S3_SECRET set in your env as login credentials.
+
+The Trinio UI will be available at http://127.0.0.1:8080/ui/. Login as `admin`. Details can be found there on queries. This is particularly useful for troubleshooting failures.
+
+For command line interactions with Trino install the CLI from https://trino.io/docs/current/installation/cli.html and follow instructions there. Use the following to login ::
+
+    trino --server localhost:8080 --catalog hive --schema acct10001 --user admin --debug
+
+Example usage ::
+
+    SHOW tables;
+    SELECT * from aws_line_items WHERE source='{source}' AND year='2021' AND month='02' LIMIT 100;
+
 Developing with OpenShift
 -------------------------
 

README.rst

@@ -30,6 +30,7 @@ DECLARE
     latest_migrations jsonb;
     required_tables int := 0;
     do_migrations boolean := false;
+    objects_exist boolean := false;
 BEGIN
     /*
      * Verify that the necessary tables are present
@@ -68,6 +69,27 @@ BEGIN
                     else schema_name
                END::text
     LOOP
+        /* Check for race condition if someone deletes a source, etc during processing */
+        SELECT EXISTS (
+                        SELECT c.oid
+                          FROM pg_class c
+                          JOIN pg_namespace n
+                            ON n.oid = c.relnamespace
+                         WHERE c.relname = 'django_migrations'
+                           AND n.nspname = schema_rec.schema_name
+                      )::boolean
+          INTO objects_exist;
+
+        IF NOT objects_exist
+        THEN
+            RAISE WARNING 'Object %.% does not exist. Skipping schema %',
+                          schema_rec.schema_name,
+                          'django_migrations',
+                          schema_rec.schema_name;
+        END IF;
+
+        CONTINUE WHEN NOT objects_exist;
+
         /* Get the latest recorded migrations by app for this tenant schema */
         IF _verbose
         THEN

db_functions/app_needs_migrations_func.sql

@@ -42,7 +42,7 @@ services:
         - prometheus_multiproc_dir=/tmp
         - API_PATH_PREFIX=${API_PATH_PREFIX-/api/cost-management}
         - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}
-        - DEMO_ACCOUNTS=${DEMO_ACCOUNTS-{}}
+        - DEMO_ACCOUNTS
         - ACCOUNT_ENHANCED_METRICS=${ACCOUNT_ENHANCED_METRICS-False}
         - RUN_GUNICORN=${RUN_GUNICORN}
       privileged: true
@@ -125,13 +125,24 @@ services:
     volumes:
       - ./pg_data:/var/lib/${DATABASE_DATA_DIR-postgresql}/data
       - ./pg_init:/docker-entrypoint-initdb.d
+      # PostgreSQL SSL: Un-comment the next two lines after running ./scripts/genssc to use sslmode in Postgres
+      # - ./certs/koku.crt:/etc/certs/koku.crt
+      # - ./certs/private/koku.key:/etc/certs/private/koku.key
     command:
       # This command give more precise control over the parameter settings
       # Included are loading the pg_stat_statements lib
       # as well as autovacuum settings that are designed to make more efficient
       # use of the autovacuum processes
       # The pg_init mount is still needed to enable the necesary extensions.
       - postgres
+      # PostgreSQL SSL: Run ./scripts/genssc to create self-signed cert and key files to support ssl
+      #                 Un-comment the three options below to utilize ssl mode and use the certificate and key
+      # - -c
+      # - ssl=on
+      # - -c
+      # - ssl_cert_file=/etc/certs/koku.crt
+      # - -c
+      # - ssl_key_file=/etc/certs/private/koku.key
       - -c
       - max_connections=1710
       - -c
@@ -248,7 +259,7 @@ services:
         - KOKU_CELERY_ENABLE_SENTRY
         - KOKU_CELERY_SENTRY_DSN
         - KOKU_SENTRY_ENVIRONMENT
-        - DEMO_ACCOUNTS=${DEMO_ACCOUNTS-{}}
+        - DEMO_ACCOUNTS
         - INITIAL_INGEST_OVERRIDE=${INITIAL_INGEST_OVERRIDE-False}
         - INITIAL_INGEST_NUM_MONTHS=${INITIAL_INGEST_NUM_MONTHS-2}
         - AUTO_DATA_INGEST=${AUTO_DATA_INGEST-True}
@@ -358,6 +369,7 @@ services:
         - SOURCES_KAFKA_PORT=${SOURCES_KAFKA_PORT-29092}
         - KOKU_SOURCES_CLIENT_PORT=${KOKU_SOURCES_CLIENT_PORT-9000}
         - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}
+        - DEMO_ACCOUNTS
         - prometheus_multiproc_dir=/tmp
       privileged: true
       ports:

docker-compose.yml

@@ -11,8 +11,8 @@ azure-mgmt-storage==11.2.0
 azure-storage-blob==12.7.1
 beautifulsoup4==4.9.3
 billiard==3.6.3.0
-boto3==1.17.8
-botocore==1.20.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+boto3==1.17.12
+botocore==1.20.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 bs4==0.0.1
 cachetools==4.2.1
 celery-prometheus-exporter==1.7.0
@@ -23,23 +23,23 @@ chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2
 ciso8601==2.1.3
 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 confluent-kafka==1.6.0
-cryptography==3.4.5; python_version >= '3.6'
+cryptography==3.4.6; python_version >= '3.6'
 django-cors-headers==3.7.0
 django-environ==0.4.5
 django-extensions==3.1.1
 django-filter==2.4.0
 django-prometheus==2.1.0
 django-redis==4.12.1
 django-tenant-schemas==1.10.0
-django==3.1.6
+django==3.1.7
 djangorestframework-csv==2.1.0
 djangorestframework==3.12.2
 flake8==3.8.4
 google-api-core[grpc]==1.26.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 google-api-python-client==1.12.8
 google-auth-httplib2==0.0.4
-google-auth==1.26.1
-google-cloud-bigquery==2.8.0
+google-auth==1.27.0
+google-cloud-bigquery==2.9.0
 google-cloud-core==1.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 google-crc32c==1.1.2; python_version >= '3.5'
 google-resumable-media==1.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
@@ -69,7 +69,7 @@ pint==0.16.1
 presto-python-client==0.7.0
 prometheus-client==0.9.0
 proto-plus==1.13.0; python_version >= '3.6'
-protobuf==3.14.0
+protobuf==3.15.1
 psutil==5.8.0
 psycopg2-binary==2.8.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pyarrow==3.0.0
@@ -88,8 +88,8 @@ requests-oauthlib==1.3.0
 requests==2.25.1
 rsa==4.7.1; python_version >= '3.6'
 s3transfer==0.3.4
-scipy==1.6.0; python_version >= '3.7'
-sentry-sdk==0.20.2
+scipy==1.6.1; python_version >= '3.7'
+sentry-sdk==0.20.3
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 soupsieve==2.2; python_version >= '3.0'
 sqlparse==0.4.1; python_version >= '3.5'