Setup CockroachDB in docker container
Create docker volume
Create docker volume on each host:
[root@host1 ~]# docker volume create --driver local --opt type=ext4 --opt device=/dev/nvme2n1 vol1
[root@host1 ~]# mkfs.ext4 /dev/nvme2n1
[root@host1 ~]# docker inspect vol1
[
{
"CreatedAt": "2022-09-13T01:10:50Z",
"Driver": "local",
"Labels": {},
"Mountpoint": "/var/lib/docker/volumes/vol1/_data",
"Name": "vol1",
"Options": {
"device": "/dev/nvme2n1",
"type": "ext4"
},
"Scope": "local"
}
]
Start CockroachDB instance
Start the CockroachDB instance on each host:
[root@host1 ~]# docker run -d --cpus="0.25" --memory="32g" --name=roach1 --hostname=host1 -p 26257:26257 -p 8080:8080 -v "vol1:/cockroach/cockroach-data" cockroachdb/cockroach:v22.1.6 start --insecure --join=host1,host2,host3
[root@host2 ~]# docker run -d --cpus="0.25" --memory="32g" --name=roach2 --hostname=host2 -p 26257:26257 -p 8080:8080 -v "vol2:/cockroach/cockroach-data" cockroachdb/cockroach:v22.1.6 start --insecure --join=host1,host2,host3
[root@host3 ~]# docker run -d --cpus="0.25" --memory="32g" --name=roach3 --hostname=host3 -p 26257:26257 -p 8080:8080 -v "vol3:/cockroach/cockroach-data" cockroachdb/cockroach:v22.1.6 start --insecure --join=host1,host2,host3
Initialize the CockroachDB cluster
Initialize the cluster on any of the cluster nodes:
[root@host1 ~]# docker exec -it roach1 ./cockroach init --insecure
Check the cluster status:
[root@host1 ~]# docker exec -it roach1 ./cockroach node ls --insecure
id
------
1
2
3
(3 rows)
[root@host1 ~]# docker exec -it roach1 ./cockroach node status --insecure
id | address | sql_address | build | started_at | updated_at | locality | is_available | is_live
-----+------------------+------------------+---------+----------------------------+----------------------------+----------+--------------+----------
1 | host1:26257 | host1:26257 | v22.1.6 | 2022-09-13 01:09:19.857864 | 2022-09-13 01:10:22.870301 | | true | true
2 | host2:26257 | host2:26257 | v22.1.6 | 2022-09-13 01:09:20.668787 | 2022-09-13 01:10:23.671303 | | true | true
3 | host3:26257 | host3:26257 | v22.1.6 | 2022-09-13 01:09:20.910602 | 2022-09-13 01:10:19.415254 | | true | true
(3 rows)
[root@host1 ~]# docker exec -it roach1 df -h
Filesystem Size Used Avail Use% Mounted on
overlay 50G 7.1G 43G 15% /
tmpfs 64M 0 64M 0% /dev
tmpfs 504G 0 504G 0% /sys/fs/cgroup
shm 64M 0 64M 0% /dev/shm
/dev/nvme2n1 1.5T 1.6G 1.4T 1% /cockroach/cockroach-data
/dev/mapper/centos-root 50G 7.1G 43G 15% /etc/hosts
tmpfs 504G 0 504G 0% /proc/acpi
tmpfs 504G 0 504G 0% /proc/scsi
tmpfs 504G 0 504G 0% /sys/firmware
[root@host1 ~]# mount | grep nvme2n1
/dev/nvme2n1 on /var/lib/docker/volumes/vol1/_data type ext4 (rw,relatime)
Check the cluster startup logs
[root@host1 ~]# docker exec -it roach1 grep 'node starting' cockroach-data/logs/cockroach.log -A 11
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +CockroachDB node starting at 2022-09-13 00:57:22.532397342 +0000 UTC (took 110.6s)
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +build: CCL v22.1.6 @ 2022/08/23 17:05:04 (go1.17.11)
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +webui: ‹http://host1:8080›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +sql: ‹postgresql://root@host1:26257/defaultdb?sslmode=disable›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +sql (JDBC): ‹jdbc:postgresql://host1:26257/defaultdb?sslmode=disable&user=root›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +RPC client flags: ‹/cockroach/cockroach <client cmd> --host=host1:26257 --insecure›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +logs: ‹/cockroach/cockroach-data/logs›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +temp dir: ‹/cockroach/cockroach-data/cockroach-temp1471904785›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +external I/O path: ‹/cockroach/cockroach-data/extern›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +store[0]: ‹path=/cockroach/cockroach-data›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +storage engine: pebble
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +clusterID: ‹8276f8fb-30f2-4712-ad4e-f008d382372d›
Load the tpcc dataset
[root@host1 ~]# docker exec -it roach1 ./cockroach workload init tpcc "postgresql://root@host1:26257?sslmode=disable" --warehouses 2500 --drop
[root@host2 ~]# iostat -ktdx 2 nvme2n1
09/13/2022 01:14:00 AM
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
nvme2n1 0.00 239.50 0.00 2112.00 0.00 167118.00 158.26 0.27 0.13 0.00 0.13 0.25 53.00
09/13/2022 01:14:02 AM
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
nvme2n1 0.00 268.00 0.00 2369.50 0.00 185266.00 156.38 0.30 0.13 0.00 0.13 0.24 56.50
The following are the elapsed time to load dataset for 2500 warehouses.
I220913 04:28:20.633547 1 workload/workloadsql/dataload.go:146 [-] 1 imported warehouse (0s, 2500 rows)
I220913 04:28:20.900607 1 workload/workloadsql/dataload.go:146 [-] 2 imported district (0s, 25000 rows)
I220913 04:46:27.360882 7 1@util/log/log_flush.go:99 [-] 3 hangup received, flushing logs
I220913 04:59:37.468628 1 workload/workloadsql/dataload.go:146 [-] 4 imported customer (31m17s, 75000000 rows)
I220913 05:06:54.166160 1 workload/workloadsql/dataload.go:146 [-] 5 imported history (7m17s, 75000000 rows)
I220913 09:17:27.487420 1 workload/workloadsql/dataload.go:146 [-] 6 imported order (4h10m33s, 75000000 rows)
I220913 09:18:29.376933 1 workload/workloadsql/dataload.go:146 [-] 7 imported new_order (1m2s, 22500000 rows)
I220913 09:18:29.975510 1 workload/workloadsql/dataload.go:146 [-] 8 imported item (1s, 100000 rows)
I220913 10:07:29.447874 1 workload/workloadsql/dataload.go:146 [-] 9 imported stock (48m59s, 250000000 rows)
I220913 11:18:52.140400 1 workload/workloadsql/dataload.go:146 [-] 10 imported order_line (1h11m23s, 750022630 rows)
The following shows the used file system size for the 2500 warehouses.
[root@host1 ~]# docker exec -it roach1 df -h
Filesystem Size Used Avail Use% Mounted on
overlay 50G 7.1G 43G 15% /
tmpfs 64M 0 64M 0% /dev
tmpfs 504G 0 504G 0% /sys/fs/cgroup
shm 64M 0 64M 0% /dev/shm
/dev/nvme2n1 1.5T 49G 1.4T 4% /cockroach/cockroach-data
/dev/mapper/centos-root 50G 7.1G 43G 15% /etc/hosts
tmpfs 504G 0 504G 0% /proc/acpi
tmpfs 504G 0 504G 0% /proc/scsi
tmpfs 504G 0 504G 0% /sys/firmware
Run the tpcc workload
Run tpcc workload from remote host:
[root@host4 ~]# cat addr
postgres://root@host1:26257?sslmode=disable postgres://root@host2:26257?sslmode=disable postgres://root@host3:26257?sslmode=disable
[root@host4 ~]# cockroach workload run tpcc --warehouses=2500 --ramp=5m --duration=30m $(cat addr)