Skip to content

Commit 0a76f5e

Browse files
authored
Merge pull request #2724 from project-koku/rc-2021.03.15
Rc 2021.03.15
2 parents 172729a + 935e72a commit 0a76f5e

89 files changed

Lines changed: 1646 additions & 534 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env.example

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,15 @@ POSTGRES_SQL_SERVICE_PORT=15432
99
DATABASE_USER=postgres
1010
DATABASE_ADMIN=postgres
1111
DATABASE_PASSWORD=postgres
12+
HIVE_DATABASE_NAME=hive
13+
HIVE_DATABASE_USER=hive
14+
HIVE_DATABASE_PASSWORD=hive
1215
DEVELOPMENT=True
1316
prometheus_multiproc_dir='/tmp'
1417
AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY
1518
AWS_SECRET_ACCESS_KEY=AWS_SECRET_KEY
1619

20+
USE_RABBIT=False
1721
ENABLE_S3_ARCHIVING=True
1822
ENABLE_PARQUET_PROCESSING=True
1923
S3_BUCKET_NAME=koku-bucket

.github/workflows/unittests.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
jobs:
1010
build:
1111

12-
runs-on: ubuntu-18.04
12+
runs-on: ubuntu-20.04
1313
strategy:
1414
max-parallel: 4
1515
matrix:
@@ -75,6 +75,7 @@ jobs:
7575
POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
7676
ACCOUNT_ENHANCED_METRICS: True
7777
prometheus_multiproc_dir: /tmp
78+
HIVE_DATABASE_PASSWORD: hivedbpw
7879

7980
- name: Convert coverage report to XML
8081
run: pipenv run coverage xml

Pipfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ kombu = ">=4.6.10,<5.0"
4949
importlib-metadata = "*"
5050
presto-python-client = ">=0.7.0"
5151
statsmodels = ">=0.12"
52+
ibm-cloud-sdk-core = ">=3.5.2"
53+
ibm-platform-services = ">=0.17.8"
5254

5355
[dev-packages]
5456
astroid = ">=2.3"

Pipfile.lock

Lines changed: 217 additions & 201 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

db_functions/app_needs_migrations_func.sql

Lines changed: 101 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,19 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
1919
-- leaf_migrations (jsonb) = leaf migration names by app from the django code
2020
-- Ex: '{<django-app>: <latest-leaf-migration-name>}'
2121
-- Set _verbose to true to see notices raised during execution
22-
DROP FUNCTION IF EXISTS public.app_needs_migrations(jsonb, boolean);
22+
DROP FUNCTION IF EXISTS public.migrations_complete(jsonb, boolean);
2323

24-
CREATE OR REPLACE FUNCTION public.app_needs_migrations(leaf_migrations jsonb, _verbose boolean DEFAULT false)
24+
CREATE OR REPLACE FUNCTION public.migrations_complete(leaf_migrations jsonb, _verbose boolean DEFAULT false)
2525
RETURNS boolean AS $BODY$
2626
DECLARE
2727
schema_rec record;
2828
leaf_app_key text;
2929
leaf_app_keys text[];
3030
latest_migrations jsonb;
3131
required_tables int := 0;
32-
do_migrations boolean := false;
33-
objects_exist boolean := false;
32+
completed_migrations boolean := true;
33+
exists_rec record;
34+
chk_res boolean;
3435
BEGIN
3536
/*
3637
* Verify that the necessary tables are present
@@ -48,7 +49,11 @@ BEGIN
4849
*/
4950
IF required_tables != 2
5051
THEN
51-
RETURN true;
52+
IF _verbose
53+
THEN
54+
RAISE WARNING 'Schema "public" not initialized';
55+
END IF;
56+
RETURN false;
5257
END IF;
5358

5459
/*
@@ -61,85 +66,114 @@ BEGIN
6166
FOR schema_rec IN
6267
SELECT t.schema_name
6368
FROM public.api_tenant t
64-
JOIN pg_namespace n
65-
on n.nspname = t.schema_name
6669
ORDER
6770
BY case when schema_name = 'public'
6871
then '0public'
6972
else schema_name
7073
END::text
7174
LOOP
72-
/* Check for race condition if someone deletes a source, etc during processing */
73-
SELECT EXISTS (
74-
SELECT c.oid
75-
FROM pg_class c
76-
JOIN pg_namespace n
77-
ON n.oid = c.relnamespace
78-
WHERE c.relname = 'django_migrations'
79-
AND n.nspname = schema_rec.schema_name
80-
)::boolean
81-
INTO objects_exist;
82-
83-
IF NOT objects_exist
75+
/* Get the latest recorded migrations by app for this tenant schema */
76+
IF _verbose
8477
THEN
85-
RAISE WARNING 'Object %.% does not exist. Skipping schema %',
86-
schema_rec.schema_name,
87-
'django_migrations',
88-
schema_rec.schema_name;
78+
RAISE INFO 'Checking migration state in schema %', schema_rec.schema_name;
8979
END IF;
9080

91-
CONTINUE WHEN NOT objects_exist;
81+
/* Check for race condition if someone deletes a source, etc during processing */
82+
EXECUTE 'SELECT EXISTS ( ' ||
83+
'SELECT c.oid ' ||
84+
'FROM pg_class c ' ||
85+
'JOIN pg_namespace n ' ||
86+
'ON n.oid = c.relnamespace ' ||
87+
'WHERE c.relname = ''django_migrations'' ' ||
88+
'AND n.nspname = ' || quote_literal(schema_rec.schema_name) || ' ' ||
89+
')::boolean as "objects_exist", ' ||
90+
'EXISTS ( ' ||
91+
'SELECT t.id ' ||
92+
'FROM public.api_tenant t ' ||
93+
'WHERE schema_name = ' || quote_literal(schema_rec.schema_name) || ' ' ||
94+
')::boolean as "tenant_exists", ' ||
95+
'EXISTS ( ' ||
96+
'SELECT n.oid ' ||
97+
'FROM pg_namespace n ' ||
98+
'WHERE nspname = ' || quote_literal(schema_rec.schema_name) || ' ' ||
99+
')::boolean as "schema_exists" '
100+
INTO exists_rec;
92101

93-
/* Get the latest recorded migrations by app for this tenant schema */
94-
IF _verbose
102+
IF exists_rec.tenant_exists AND NOT exists_rec.schema_exists
95103
THEN
96-
RAISE NOTICE 'Checking %', schema_rec.schema_name;
104+
RAISE EXCEPTION 'MIGRATION CHECK :: Tenant "%" exists, but there is no database schema.',
105+
schema_rec.schema_name;
97106
END IF;
98-
EXECUTE 'SELECT jsonb_object_agg(app, migration) ' ||
99-
'FROM ( ' ||
100-
'SELECT app, ' ||
101-
'max(name) as "migration" ' ||
102-
'FROM ' || quote_ident(schema_rec.schema_name) || '.django_migrations ' ||
103-
'GROUP BY app ' ||
104-
') AS x ;'
105-
INTO latest_migrations;
106-
107-
/* Loop through leaf apps */
108-
FOREACH leaf_app_key IN ARRAY leaf_app_keys
109-
LOOP
110-
/* test that app exists or not */
111-
IF latest_migrations ? leaf_app_key
112-
THEN
113-
/* App exists! Test if the leaf migration name is greater than the last recorded migration for the app */
114-
IF _verbose
115-
THEN
116-
RAISE NOTICE ' checking %.% > %.%',
117-
leaf_app_key,
118-
leaf_migrations->>leaf_app_key,
119-
leaf_app_key,
120-
latest_migrations->>leaf_app_key;
121-
END IF;
122-
IF leaf_migrations->>leaf_app_key > latest_migrations->>leaf_app_key
123-
THEN
124-
/* leaf ahead of last recorded. run migrations! */
125-
do_migrations = true;
126-
EXIT;
127-
END IF;
128-
ELSE
129-
/* App does not exist, run migrations! */
130-
IF _verbose
107+
IF NOT exists_rec.tenant_exists AND exists_rec.schema_exists
108+
THEN
109+
RAISE EXCEPTION 'MIGRATION CHECK :: Schema "%" exists, but there is no tenant record.',
110+
schema_rec.schema_name;
111+
END IF;
112+
113+
CONTINUE WHEN (NOT exists_rec.tenant_exists) OR (NOT exists_rec.schema_exists);
114+
115+
IF NOT exists_rec.objects_exist
116+
THEN
117+
RAISE WARNING ' %.django_migrations does not exist', schema_rec.schema_name;
118+
completed_migrations = false;
119+
ELSE
120+
EXECUTE 'SELECT jsonb_object_agg(app, migration) ' ||
121+
'FROM ( ' ||
122+
'SELECT app, ' ||
123+
'max(name) as "migration" ' ||
124+
'FROM ' || quote_ident(schema_rec.schema_name) || '.django_migrations ' ||
125+
'GROUP BY app ' ||
126+
') AS x ;'
127+
INTO latest_migrations;
128+
129+
/* Loop through leaf apps */
130+
FOREACH leaf_app_key IN ARRAY leaf_app_keys
131+
LOOP
132+
/* test that app exists or not */
133+
IF latest_migrations ? leaf_app_key
131134
THEN
132-
RAISE NOTICE ' app % missing from schema', leaf_app_key;
135+
/* App exists! Test if the leaf migration name is greater than the last recorded migration for the app */
136+
chk_res = (leaf_migrations->>leaf_app_key > latest_migrations->>leaf_app_key)::boolean;
137+
IF _verbose
138+
THEN
139+
RAISE INFO ' checking %.% > %.% (%)',
140+
leaf_app_key,
141+
leaf_migrations->>leaf_app_key,
142+
leaf_app_key,
143+
latest_migrations->>leaf_app_key,
144+
chk_res::text;
145+
END IF;
146+
IF chk_res
147+
THEN
148+
/* leaf ahead of last recorded. run migrations! */
149+
completed_migrations = false;
150+
IF _verbose
151+
THEN
152+
RAISE INFO ' Will run migrations.';
153+
END IF;
154+
END IF;
155+
ELSE
156+
/* App does not exist, run migrations! */
157+
IF _verbose
158+
THEN
159+
RAISE INFO ' app % missing from schema', leaf_app_key;
160+
END IF;
161+
completed_migrations = false;
133162
END IF;
134-
do_migrations = true;
135-
EXIT;
136-
END IF;
137-
END LOOP;
163+
164+
EXIT WHEN not completed_migrations;
165+
END LOOP;
166+
END IF;
138167

139168
/* Stop processing if we are going to run migrations */
140-
EXIT WHEN do_migrations;
169+
EXIT WHEN not completed_migrations;
141170
END LOOP;
142171

143-
RETURN do_migrations;
172+
IF _verbose
173+
THEN
174+
RAISE INFO 'Migration Check: App should%execute migrations.', case when completed_migrations then ' not ' else ' ' end::text;
175+
END IF;
176+
177+
RETURN completed_migrations;
144178
END;
145179
$BODY$ LANGUAGE PLPGSQL;

docker-compose.yml

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ services:
3737
- MASU_SERVICE_PORT=${MASU_SERVICE_PORT-8000}
3838
- RABBITMQ_HOST=${RABBITMQ_HOST-koku-rabbit}
3939
- RABBITMQ_PORT=5672
40+
- USE_RABBIT=${USE_RABBIT}
4041
- RBAC_CACHE_TTL
4142
- MASU_SECRET_KEY=abc
4243
- prometheus_multiproc_dir=/tmp
@@ -88,6 +89,7 @@ services:
8889
- MASU_SERVICE_PORT=${MASU_SERVICE_PORT-8000}
8990
- RABBITMQ_HOST=${RABBITMQ_HOST-koku-rabbit}
9091
- RABBITMQ_PORT=5672
92+
- USE_RABBIT=${USE_RABBIT}
9193
- RBAC_CACHE_TTL
9294
- MASU_SECRET_KEY=abc
9395
- prometheus_multiproc_dir=/tmp
@@ -126,8 +128,8 @@ services:
126128
- ./pg_data:/var/lib/${DATABASE_DATA_DIR-postgresql}/data
127129
- ./pg_init:/docker-entrypoint-initdb.d
128130
# PostgreSQL SSL: Un-comment the next two lines after running ./scripts/genssc to use sslmode in Postgres
129-
# - ./certs/koku.crt:/etc/certs/koku.crt
130-
# - ./certs/private/koku.key:/etc/certs/private/koku.key
131+
# - ${HOME}/.koku-db-certs/koku.crt:/etc/certs/koku.crt
132+
# - ${HOME}/.koku-db-certs/private/koku.key:/etc/certs/private/koku.key
131133
command:
132134
# This command give more precise control over the parameter settings
133135
# Included are loading the pg_stat_statements lib
@@ -241,6 +243,7 @@ services:
241243
- DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres}
242244
- RABBITMQ_HOST=${RABBITMQ_HOST-koku-rabbit}
243245
- RABBITMQ_PORT=5672
246+
- USE_RABBIT=${USE_RABBIT}
244247
- DEVELOPMENT=${DEVELOPMENT-True}
245248
- LOG_LEVEL=INFO
246249
- DJANGO_SETTINGS_MODULE=koku.settings
@@ -309,6 +312,7 @@ services:
309312
- DATABASE_PORT=5432
310313
- RABBITMQ_HOST=${RABBITMQ_HOST-koku-rabbit}
311314
- RABBITMQ_PORT=5672
315+
- USE_RABBIT=${USE_RABBIT}
312316
- DATABASE_USER=${DATABASE_USER-postgres}
313317
- DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres}
314318
- MASU_SECRET_KEY=abc
@@ -365,6 +369,8 @@ services:
365369
- SOURCES_API_PORT=${SOURCES_API_PORT-3000}
366370
- RABBITMQ_HOST=${RABBITMQ_HOST-koku-rabbit}
367371
- RABBITMQ_PORT=5672
372+
- REDIS_HOST=${REDIS_HOST-redis}
373+
- REDIS_PORT=${REDIS_PORT-6379}
368374
- SOURCES_KAFKA_HOST=${SOURCES_KAFKA_HOST-kafka}
369375
- SOURCES_KAFKA_PORT=${SOURCES_KAFKA_PORT-29092}
370376
- KOKU_SOURCES_CLIENT_PORT=${KOKU_SOURCES_CLIENT_PORT-9000}
@@ -379,6 +385,7 @@ services:
379385
- '.:/koku'
380386
links:
381387
- db
388+
- redis
382389
depends_on:
383390
- koku-base
384391

@@ -401,6 +408,7 @@ services:
401408
- DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres}
402409
- RABBITMQ_HOST=${RABBITMQ_HOST-koku-rabbit}
403410
- RABBITMQ_PORT=5672
411+
- USE_RABBIT=${USE_RABBIT}
404412
- LOG_LEVEL=INFO
405413
- DJANGO_SETTINGS_MODULE=koku.settings
406414
- MASU_SECRET_KEY=abc

docs/rtd_requirements.txt

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@ azure-common==1.1.26
66
azure-core==1.11.0
77
azure-mgmt-core==1.2.2
88
azure-mgmt-costmanagement==0.2.0
9-
azure-mgmt-resource==15.0.0
9+
azure-mgmt-resource==16.0.0
1010
azure-mgmt-storage==11.2.0
1111
azure-storage-blob==12.7.1
1212
beautifulsoup4==4.9.3
1313
billiard==3.6.3.0
14-
boto3==1.17.12
15-
botocore==1.20.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
14+
boto3==1.17.17
15+
botocore==1.20.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
1616
bs4==0.0.1
1717
cachetools==4.2.1
1818
celery-prometheus-exporter==1.7.0
@@ -39,16 +39,18 @@ google-api-core[grpc]==1.26.0; python_version >= '2.7' and python_version not in
3939
google-api-python-client==1.12.8
4040
google-auth-httplib2==0.0.4
4141
google-auth==1.27.0
42-
google-cloud-bigquery==2.9.0
42+
google-cloud-bigquery==2.10.0
4343
google-cloud-core==1.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
4444
google-crc32c==1.1.2; python_version >= '3.5'
4545
google-resumable-media==1.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
46-
googleapis-common-protos==1.52.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
47-
grpcio==1.35.0
46+
googleapis-common-protos==1.53.0; python_version >= '3.6'
47+
grpcio==1.36.0
4848
gunicorn==20.0.4
4949
httplib2==0.19.0
50+
ibm-cloud-sdk-core==3.5.2
51+
ibm-platform-services==0.17.12
5052
idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
51-
importlib-metadata==3.4.0
53+
importlib-metadata==3.7.0
5254
isodate==0.6.0
5355
jinja2==2.11.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
5456
jinjasql==0.1.8
@@ -68,8 +70,8 @@ patsy==0.5.1
6870
pint==0.16.1
6971
presto-python-client==0.7.0
7072
prometheus-client==0.9.0
71-
proto-plus==1.13.0; python_version >= '3.6'
72-
protobuf==3.15.1
73+
proto-plus==1.14.2; python_version >= '3.6'
74+
protobuf==3.15.3
7375
psutil==5.8.0
7476
psycopg2-binary==2.8.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
7577
pyarrow==3.0.0
@@ -86,7 +88,7 @@ querystring-parser==1.2.4
8688
redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
8789
requests-oauthlib==1.3.0
8890
requests==2.25.1
89-
rsa==4.7.1; python_version >= '3.6'
91+
rsa==4.7.2; python_version >= '3.6'
9092
s3transfer==0.3.4
9193
scipy==1.6.1; python_version >= '3.7'
9294
sentry-sdk==0.20.3

0 commit comments

Comments
 (0)