]>
Commit | Line | Data |
---|---|---|
aee94f69 TL |
1 | #!/bin/bash |
2 | ||
3 | root_dir() | |
4 | { | |
5 | cd $(git rev-parse --show-toplevel) | |
6 | } | |
7 | ||
8 | modify_end_point_on_hive_properties() | |
9 | { | |
10 | #not in use | |
11 | return; | |
12 | #TODO if ./trino/catalog/hive.properties exist | |
13 | ||
14 | [ $# -lt 1 ] && echo type s3-endpoint-url && return | |
15 | root_dir | |
16 | export S3_ENDPOINT=$1 | |
17 | cat container/trino/trino/catalog/hive.properties | awk -v x=${S3_ENDPOINT:-NO_SET} '{if(/hive.s3.endpoint/){print "hive.s3.endpoint="x"\n";} else {print $0;}}' > /tmp/hive.properties | |
18 | cp /tmp/hive.properties container/trino/trino/catalog/hive.properties | |
19 | cat ./container/trino/hms_trino.yaml | awk -v x=${S3_ENDPOINT:-NOT_SET} '{if(/[ *]- S3_ENDPOINT/){print "\t- S3_ENDPOINT="x"\n";} else {print $0;}}' > /tmp/hms_trino.yaml | |
20 | cp /tmp/hms_trino.yaml ./container/trino/hms_trino.yaml | |
21 | cd - | |
22 | } | |
23 | ||
24 | trino_exec_command() | |
25 | { | |
26 | ## run SQL statement on trino | |
27 | sudo docker exec -it trino /bin/bash -c "time trino --catalog hive --schema cephs3 --execute \"$@\"" | |
28 | } | |
29 | ||
30 | boot_trino_hms() | |
31 | { | |
32 | root_dir | |
33 | [ -z ${S3_ENDPOINT} ] && echo "missing end-variable S3_ENDPOINT (URL)" && return | |
34 | [ -z ${S3_ACCESS_KEY} ] && echo missing end-variable S3_ACCESS_KEY && return | |
35 | [ -z ${S3_SECRET_KEY} ] && echo missing end-variable S3_SECRET_KEY && return | |
36 | ||
37 | # modify hms_trino.yaml according to user setup (environment variables) | |
38 | cat ./container/trino/hms_trino.yaml | \ | |
39 | awk -v x=${S3_ENDPOINT:-NOT_SET} '{if(/- S3_ENDPOINT/){print " - S3_ENDPOINT="x;} else {print $0;}}' | \ | |
40 | awk -v x=${S3_ACCESS_KEY:-NOT_SET} '{if(/- S3_ACCESS_KEY/){print " - S3_ACCESS_KEY="x;} else {print $0;}}' | \ | |
41 | awk -v x=${S3_SECRET_KEY:-NOT_SET} '{if(/- S3_SECRET_KEY/){print " - S3_SECRET_KEY="x;} else {print $0;}}' > /tmp/hms_trino.yaml | |
42 | cp /tmp/hms_trino.yaml ./container/trino/hms_trino.yaml | |
43 | ||
44 | ||
45 | ||
46 | # modify hive.properties according to user setup (environment variables) | |
47 | cat container/trino/trino/catalog/hive.properties | \ | |
48 | awk -v x=${S3_ENDPOINT:-NO_SET} '{if(/hive.s3.endpoint/){print "hive.s3.endpoint="x"\n";} else {print $0;}}' | \ | |
49 | awk -v x=${S3_ACCESS_KEY:-NO_SET} '{if(/hive.s3.aws-access-key/){print "hive.s3.aws-access-key="x;} else {print $0;}}' | \ | |
50 | awk -v x=${S3_SECRET_KEY:-NO_SET} '{if(/hive.s3.aws-secret-key/){print "hive.s3.aws-secret-key="x;} else {print $0;}}' > /tmp/hive.properties | |
51 | cp /tmp/hive.properties ./container/trino/trino/catalog/hive.properties | |
52 | ||
53 | sudo docker compose -f ./container/trino/hms_trino.yaml up -d | |
54 | cd - | |
55 | } | |
56 | ||
57 | shutdown_trino_hms() | |
58 | { | |
59 | root_dir | |
60 | sudo docker compose -f ./container/trino/hms_trino.yaml down | |
61 | cd - | |
62 | } | |
63 | ||
64 | trino_create_table() | |
65 | { | |
66 | table_name=$1 | |
67 | create_table_comm="create table hive.cephs3.${table_name}(c1 varchar,c2 varchar,c3 varchar,c4 varchar, c5 varchar,c6 varchar,c7 varchar,c8 varchar,c9 varchar,c10 varchar) | |
68 | WITH ( external_location = 's3a://hive/warehouse/cephs3/${table_name}/',format = 'TEXTFILE',textfile_field_separator = ',');" | |
69 | sudo docker exec -it trino /bin/bash -c "trino --catalog hive --schema cephs3 --execute \"${create_table_comm}\"" | |
70 | } | |
71 | ||
72 | tpcds_cli() | |
73 | { | |
74 | ## a CLI example for generating TPCDS data | |
75 | sudo docker run --env S3_ENDPOINT=172.17.0.1:8000 --env S3_ACCESS_KEY=b2345678901234567890 --env S3_SECRET_KEY=b234567890123456789012345678901234567890 --env BUCKET_NAME=hive --env SCALE=2 -it galsl/hadoop:tpcds bash -c '/root/run_tpcds_with_scale' | |
76 | } | |
77 | ||
78 | update_table_external_location() | |
79 | { | |
80 | root_dir | |
81 | [ -z ${BUCKET_NAME} ] && echo need to define BUCKET_NAME && return | |
82 | [ -z ${SCALE} ] && echo need to define SCALE && return | |
83 | ||
84 | cat TPCDS/ddl/create_tpcds_tables.sql | sed "s/tpcds2\/4/${BUCKET_NAME}\/SCALE_${SCALE}/" | |
85 | } | |
86 |