-
Notifications
You must be signed in to change notification settings - Fork 7
/
export
executable file
·138 lines (120 loc) · 3.2 KB
/
export
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#!/usr/bin/env bash
set -euo pipefail
# Init lib.
# shellcheck source=SCRIPTDIR/../lib/vector-test-harness/init.sh
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/../lib/vector-test-harness/init.sh"
#
# Usage
#
register_usage <<END
USAGE
export
DESCRIPTION
Exports all performance tests results in JSON format. This data is also used
on the Vector website: https://vector.dev
OPTIONS
none
EXAMPLES
./export
END
#
# Dependencies
#
ensure_athena_dependencies_available
ensure_commad_available "ruby"
#
# Header
#
print_vector_logo
print_divider
cat <<END
Export
END
print_divider
print_spacer
#
# Execute
#
# EXECUTION_ID=$(athena_execute_query "MSCK REPAIR TABLE vector_tests")
EXECUTION_ID="$(athena_execute_query "$(cat <<EOF
WITH
latest_versions AS (
SELECT
name,
configuration,
subject,
MAX(version) AS latest_version
FROM vector_tests
GROUP BY name, configuration, subject
)
SELECT
name,
configuration,
subject,
version,
COUNT(*) AS count,
AVG(duration) AS duration_avg,
MAX(duration) AS duration_max,
AVG(cpu_sys_avg) AS cpu_sys_avg,
AVG(cpu_usr_avg) AS cpu_usr_avg,
AVG(cpu_sys_max) AS cpu_sys_max,
AVG(cpu_usr_max) AS cpu_usr_max,
AVG(load_avg_1m) AS load_avg_1m,
AVG(mem_used_avg) AS mem_used_avg,
AVG(mem_used_max) AS mem_used_max,
AVG(disk_read_avg) AS disk_read_avg,
AVG(disk_read_sum) AS disk_read_sum,
AVG(disk_writ_sum) AS disk_writ_sum,
AVG(io_read_sum) AS io_read_sum,
AVG(io_writ_sum) AS io_writ_sum,
AVG(net_recv_avg) AS net_recv_avg,
AVG(net_recv_sum) AS net_recv_sum,
AVG(net_send_sum) AS net_send_sum,
AVG(sock_total_sum) AS sock_total_sum,
AVG(tcp_act_sum) AS tcp_act_sum,
AVG(tcp_syn_sum) AS tcp_syn_sum,
AVG(tcp_clo_sum) AS tcp_clo_sum
FROM (
SELECT
v.name,
v.configuration,
v.subject,
v.version,
v.timestamp,
MAX(epoch) - MIN(epoch) AS duration,
AVG(cpu_sys) AS cpu_sys_avg,
AVG(cpu_usr) AS cpu_usr_avg,
MAX(cpu_sys) AS cpu_sys_max,
MAX(cpu_usr) AS cpu_usr_max,
AVG(load_avg_1m) AS load_avg_1m,
AVG(mem_used) AS mem_used_avg,
MAX(mem_used) AS mem_used_max,
(SUM(disk_read) / (MAX(epoch) - MIN(epoch))) AS disk_read_avg,
SUM(disk_read) AS disk_read_sum,
SUM(disk_writ) AS disk_writ_sum,
SUM(io_read) AS io_read_sum,
SUM(io_writ) AS io_writ_sum,
(SUM(net_recv) / (MAX(epoch) - MIN(epoch))) AS net_recv_avg,
SUM(net_recv) AS net_recv_sum,
SUM(net_send) AS net_send_sum,
SUM(sock_total) AS sock_total_sum,
SUM(tcp_act) AS tcp_act_sum,
SUM(tcp_syn) AS tcp_syn_sum,
SUM(tcp_clo) AS tcp_clo_sum
FROM vector_tests v
INNER JOIN latest_versions ON
latest_versions.name = v.name AND
latest_versions.configuration = v.configuration AND
latest_versions.subject = v.subject
AND latest_versions.latest_version = v.version
GROUP BY v.name, v.configuration, v.subject, v.version, v.timestamp
)
GROUP BY name, configuration, subject, version
ORDER BY subject ASC
EOF
)")"
RESULTS="$(athena_get_results "$EXECUTION_ID")"
print_spacer
FILE_NAME="results.json"
"$VECTOR_TEST_HARNESS_UTILS_DIR/export_json" "$RESULTS" "$FILE_NAME"
aws s3 cp "$FILE_NAME" "s3://$VECTOR_TEST_RESULTS_S3_BUCKET_NAME/summaries/latest.json"