-
Notifications
You must be signed in to change notification settings - Fork 18
/
workflow_batch.sh
115 lines (88 loc) · 2.79 KB
/
workflow_batch.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/bin/bash
## 用来自动测试不同索引下的查询性能
# 879999987
# 879999987
primary_keys=(
"uid,its"
"uid,its,action_code"
)
indexs=(
8192
16384
32768
65536
)
default_pk="uid,its,action_code"
default_index_granularity=8192
ck_conn="localhost"
trap "kill -9 0; exit 0" INT HUP QUIT TERM
ck="clickhouse-client --host ${ck_conn}"
table="t_event"
dis_table="dis_event"
file_dir=$2 ##/data/clickhouse/yiguan/Analysys_olap_2018_demo.dat
function bench() {
pk=$1
index_granularity=$2
echo "${table} pk : $pk ; index_granularity : ${index_granularity}"
echo "DROP TABLE if exists ${table} on cluster logs" | $ck
echo "DROP TABLE if exists ${dis_table} on cluster logs;" | $ck
echo "
CREATE TABLE ${table} on cluster logs
(
uid String,
its UInt32 CODEC(NONE),
action_code Enum8('evaluationGoods' = 1, 'browseGoods' = 2, 'viewOrder' = 3, 'startUp' = 4, 'unsubscribeGoods' = 5, 'login' = 6, 'order' = 7, 'reminding' = 8, 'addCart' = 9, 'consultGoods' = 10, 'searchGoods' = 11, 'shareGoods' = 12, 'orderPayment' = 13, 'collectionGoods' = 14, 'confirm' = 15),
action_name String,
event_city String,
event_name String,
event_brand String,
event_price Float32 CODEC(NONE),
event_nums Int8,
event_how Int8,
day Date
)
ENGINE = MergeTree PARTITION BY toYYYYMMDD(day) ORDER BY ($pk) SETTINGS index_granularity = ${index_granularity}" | $ck
## metroHash64
echo "CREATE TABLE ${dis_table} on cluster logs as ${table} ENGINE = Distributed(logs, default, t_event, metroHash64( toString(uid)) ); " | $ck
if [ $? != 0 ];then
exit 1
fi
echo "start loading data"
for file_path in `ls $file_dir | grep sort`;do
echo "load ${file_dir}/${file_path}"
../importer -dsn=tcp://${ck_conn}:9000 -file=${file_dir}/${file_path} -logAll=false -table=${dis_table} -pktype=String
done
if [ $? != 0 ];then
exit 1
fi
echo "start to sleep 100"
sleep 100
echo "start to optimize"
echo "optimize table ${table} on cluster logs final" | $ck
echo "start to benchmark"
bash bench.sh ${table} ${ck_conn}
}
#! /bin/sh -
shell_name=`basename $0 .sh`
case $1 in
pk)
echo "testing pk"
for pk in ${primary_keys[@]};do
bench "$pk" "$default_index_granularity"
done
;;
index)
echo "testing indexs"
for idx in ${indexs[@]};do
bench "$default_pk" "$idx"
done
;;
none)
echo "just testing"
bench "$default_pk" "$default_index_granularity"
;;
*)
echo "Usage: $shell_name [pk|index]"
exit 1
;;
esac