Home Setup CockroachDB in docker container
Post
Cancel

Setup CockroachDB in docker container

Create docker volume

Create docker volume on each host:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
[root@host1 ~]# docker volume create --driver local --opt type=ext4 --opt device=/dev/nvme2n1 vol1
[root@host1 ~]# mkfs.ext4 /dev/nvme2n1
[root@host1 ~]# docker inspect vol1
[
    {
        "CreatedAt": "2022-09-13T01:10:50Z",
        "Driver": "local",
        "Labels": {},
        "Mountpoint": "/var/lib/docker/volumes/vol1/_data",
        "Name": "vol1",
        "Options": {
            "device": "/dev/nvme2n1",
            "type": "ext4"
        },
        "Scope": "local"
    }
]

Start CockroachDB instance

Start the CockroachDB instance on each host:

1
2
3
4
5
[root@host1 ~]# docker run -d --cpus="0.25" --memory="32g" --name=roach1 --hostname=host1 -p 26257:26257 -p 8080:8080 -v "vol1:/cockroach/cockroach-data" cockroachdb/cockroach:v22.1.6 start --insecure --join=host1,host2,host3

[root@host2 ~]# docker run -d --cpus="0.25" --memory="32g" --name=roach2 --hostname=host2 -p 26257:26257 -p 8080:8080 -v "vol2:/cockroach/cockroach-data" cockroachdb/cockroach:v22.1.6 start --insecure --join=host1,host2,host3

[root@host3 ~]# docker run -d --cpus="0.25" --memory="32g" --name=roach3 --hostname=host3 -p 26257:26257 -p 8080:8080 -v "vol3:/cockroach/cockroach-data" cockroachdb/cockroach:v22.1.6 start --insecure --join=host1,host2,host3

Initialize the CockroachDB cluster

Initialize the cluster on any of the cluster nodes:

1
[root@host1 ~]# docker exec -it roach1 ./cockroach init --insecure

Check the cluster status:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
[root@host1 ~]# docker exec -it roach1 ./cockroach node ls --insecure
  id
------
   1
   2
   3
(3 rows)

[root@host1 ~]# docker exec -it roach1 ./cockroach node status --insecure
  id |     address      |   sql_address    |  build  |         started_at         |         updated_at         | locality | is_available | is_live
-----+------------------+------------------+---------+----------------------------+----------------------------+----------+--------------+----------
   1 | host1:26257 | host1:26257 | v22.1.6 | 2022-09-13 01:09:19.857864 | 2022-09-13 01:10:22.870301 |          | true         | true
   2 | host2:26257 | host2:26257 | v22.1.6 | 2022-09-13 01:09:20.668787 | 2022-09-13 01:10:23.671303 |          | true         | true
   3 | host3:26257 | host3:26257 | v22.1.6 | 2022-09-13 01:09:20.910602 | 2022-09-13 01:10:19.415254 |          | true         | true
(3 rows)

[root@host1 ~]# docker exec -it roach1 df -h
Filesystem               Size  Used Avail Use% Mounted on
overlay                   50G  7.1G   43G  15% /
tmpfs                     64M     0   64M   0% /dev
tmpfs                    504G     0  504G   0% /sys/fs/cgroup
shm                       64M     0   64M   0% /dev/shm
/dev/nvme2n1             1.5T  1.6G  1.4T   1% /cockroach/cockroach-data
/dev/mapper/centos-root   50G  7.1G   43G  15% /etc/hosts
tmpfs                    504G     0  504G   0% /proc/acpi
tmpfs                    504G     0  504G   0% /proc/scsi
tmpfs                    504G     0  504G   0% /sys/firmware

[root@host1 ~]# mount | grep nvme2n1
/dev/nvme2n1 on /var/lib/docker/volumes/vol1/_data type ext4 (rw,relatime)

Check the cluster startup logs

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@host1 ~]# docker exec -it roach1 grep 'node starting' cockroach-data/logs/cockroach.log -A 11
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +CockroachDB node starting at 2022-09-13 00:57:22.532397342 +0000 UTC (took 110.6s)
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +build:               CCL v22.1.6 @ 2022/08/23 17:05:04 (go1.17.11)
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +webui:               ‹http://host1:8080›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +sql:                 ‹postgresql://root@host1:26257/defaultdb?sslmode=disable›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +sql (JDBC):          ‹jdbc:postgresql://host1:26257/defaultdb?sslmode=disable&user=root›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +RPC client flags:    ‹/cockroach/cockroach <client cmd> --host=host1:26257 --insecure›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +logs:                ‹/cockroach/cockroach-data/logs›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +temp dir:            ‹/cockroach/cockroach-data/cockroach-temp1471904785›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +external I/O path:   ‹/cockroach/cockroach-data/extern›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +store[0]:            ‹path=/cockroach/cockroach-data›
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +storage engine:      pebble
I220913 00:57:22.532570 151 1@cli/start.go:1028 ⋮ [n1] 252 +clusterID:           ‹8276f8fb-30f2-4712-ad4e-f008d382372d›

Load the tpcc dataset

1
2
3
4
5
6
7
8
9
10
[root@host1 ~]# docker exec -it roach1 ./cockroach workload init tpcc "postgresql://root@host1:26257?sslmode=disable" --warehouses 2500 --drop

[root@host2 ~]# iostat -ktdx 2 nvme2n1
09/13/2022 01:14:00 AM
Device:         rrqm/s   wrqm/s     r/s     w/s    rkB/s    wkB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
nvme2n1           0.00   239.50    0.00 2112.00     0.00 167118.00   158.26     0.27    0.13    0.00    0.13   0.25  53.00

09/13/2022 01:14:02 AM
Device:         rrqm/s   wrqm/s     r/s     w/s    rkB/s    wkB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
nvme2n1           0.00   268.00    0.00 2369.50     0.00 185266.00   156.38     0.30    0.13    0.00    0.13   0.24  56.50

The following are the elapsed time to load dataset for 2500 warehouses.

1
2
3
4
5
6
7
8
9
10
I220913 04:28:20.633547 1 workload/workloadsql/dataload.go:146  [-] 1  imported warehouse (0s, 2500 rows)
I220913 04:28:20.900607 1 workload/workloadsql/dataload.go:146  [-] 2  imported district (0s, 25000 rows)
I220913 04:46:27.360882 7 1@util/log/log_flush.go:99  [-] 3  hangup received, flushing logs
I220913 04:59:37.468628 1 workload/workloadsql/dataload.go:146  [-] 4  imported customer (31m17s, 75000000 rows)
I220913 05:06:54.166160 1 workload/workloadsql/dataload.go:146  [-] 5  imported history (7m17s, 75000000 rows)
I220913 09:17:27.487420 1 workload/workloadsql/dataload.go:146  [-] 6  imported order (4h10m33s, 75000000 rows)
I220913 09:18:29.376933 1 workload/workloadsql/dataload.go:146  [-] 7  imported new_order (1m2s, 22500000 rows)
I220913 09:18:29.975510 1 workload/workloadsql/dataload.go:146  [-] 8  imported item (1s, 100000 rows)
I220913 10:07:29.447874 1 workload/workloadsql/dataload.go:146  [-] 9  imported stock (48m59s, 250000000 rows)
I220913 11:18:52.140400 1 workload/workloadsql/dataload.go:146  [-] 10  imported order_line (1h11m23s, 750022630 rows)

The following shows the used file system size for the 2500 warehouses.

1
2
3
4
5
6
7
8
9
10
11
[root@host1 ~]# docker exec -it roach1 df -h
Filesystem               Size  Used Avail Use% Mounted on
overlay                   50G  7.1G   43G  15% /
tmpfs                     64M     0   64M   0% /dev
tmpfs                    504G     0  504G   0% /sys/fs/cgroup
shm                       64M     0   64M   0% /dev/shm
/dev/nvme2n1             1.5T   49G  1.4T   4% /cockroach/cockroach-data
/dev/mapper/centos-root   50G  7.1G   43G  15% /etc/hosts
tmpfs                    504G     0  504G   0% /proc/acpi
tmpfs                    504G     0  504G   0% /proc/scsi
tmpfs                    504G     0  504G   0% /sys/firmware

Run the tpcc workload

Run tpcc workload from remote host:

1
2
3
[root@host4 ~]# cat addr
postgres://root@host1:26257?sslmode=disable postgres://root@host2:26257?sslmode=disable postgres://root@host3:26257?sslmode=disable
[root@host4 ~]# cockroach workload run tpcc --warehouses=2500 --ramp=5m --duration=30m $(cat addr)

Reference

This post is licensed under CC BY 4.0 by the author.

Using netstat to check network package retransmission

Understanding Azure disk bursting

Comments powered by Disqus.