Useful Elastic cluster APIs
Cluster settings
$ curl -q -s http://10.10.10.1:39200/_cluster/health
{
"cluster_name": "rally-benchmark",
"status": "green",
"timed_out": false,
"number_of_nodes": 3,
"number_of_data_nodes": 3,
"active_primary_shards": 8,
"active_shards": 16,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 100
}
$ curl -q -s http://10.10.10.1:39200/_cluster/settings| jq -r
{
"persistent": {},
"transient": {}
}
$ curl -q -s http://10.10.10.1:39200/_cluster/stats| jq -r
{
"_nodes": {
"total": 3,
"successful": 3,
"failed": 0
},
"cluster_name": "rally-benchmark",
"cluster_uuid": "_aPIWBOCS0Gtyo4fTp3Wrg",
"timestamp": 1667177008901,
"status": "green",
"indices": {
"count": 4,
"shards": {
"total": 16,
"primaries": 8,
"replication": 1,
"index": {
"shards": {
"min": 2,
"max": 10,
"avg": 4
},
"primaries": {
"min": 1,
"max": 5,
"avg": 2
},
"replication": {
"min": 1,
"max": 1,
"avg": 1
}
}
},
"docs": {
"count": 11396556,
"deleted": 0
},
"store": {
"size_in_bytes": 6443644350,
"total_data_set_size_in_bytes": 6443644350,
"reserved_in_bytes": 0
},
"fielddata": {
"memory_size_in_bytes": 14072,
"evictions": 0
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 3883795,
"hit_count": 0,
"miss_count": 3883795,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 219,
"memory_in_bytes": 1526988,
"terms_memory_in_bytes": 1189120,
"stored_fields_memory_in_bytes": 115320,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 155520,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 67028,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1667016435644,
"file_sizes": {}
},
"mappings": {
"field_types": [
{
"name": "boolean",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "constant_keyword",
"count": 3,
"index_count": 1,
"script_count": 0
},
{
"name": "date",
"count": 6,
"index_count": 2,
"script_count": 0
},
{
"name": "geo_point",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "integer",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "ip",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "keyword",
"count": 32,
"index_count": 3,
"script_count": 0
},
{
"name": "long",
"count": 3,
"index_count": 2,
"script_count": 0
},
{
"name": "object",
"count": 13,
"index_count": 2,
"script_count": 0
},
{
"name": "text",
"count": 17,
"index_count": 3,
"script_count": 0
}
],
"runtime_field_types": []
},
"analysis": {
"char_filter_types": [],
"tokenizer_types": [],
"filter_types": [],
"analyzer_types": [],
"built_in_char_filters": [],
"built_in_tokenizers": [],
"built_in_filters": [],
"built_in_analyzers": []
},
"versions": [
{
"version": "7.17.0",
"index_count": 4,
"primary_shard_count": 8,
"total_primary_bytes": 3221823908
}
]
},
"nodes": {
"count": {
"total": 3,
"coordinating_only": 0,
"data": 3,
"data_cold": 3,
"data_content": 3,
"data_frozen": 3,
"data_hot": 3,
"data_warm": 3,
"ingest": 3,
"master": 3,
"ml": 0,
"remote_cluster_client": 3,
"transform": 3,
"voting_only": 0
},
"versions": [
"7.17.0"
],
"os": {
"available_processors": 24,
"allocated_processors": 24,
"names": [
{
"name": "Linux",
"count": 3
}
],
"pretty_names": [
{
"pretty_name": "CentOS Linux 7 (Core)",
"count": 3
}
],
"architectures": [
{
"arch": "amd64",
"count": 3
}
],
"mem": {
"total_in_bytes": 100700000256,
"free_in_bytes": 68239511552,
"used_in_bytes": 32460488704,
"free_percent": 68,
"used_percent": 32
}
},
"process": {
"cpu": {
"percent": 0
},
"open_file_descriptors": {
"min": 460,
"max": 493,
"avg": 474
}
},
"jvm": {
"max_uptime_in_millis": 169991344,
"versions": [
{
"version": "1.8.0_352",
"vm_name": "OpenJDK 64-Bit Server VM",
"vm_version": "25.352-b08",
"vm_vendor": "Red Hat, Inc.",
"bundled_jdk": true,
"using_bundled_jdk": false,
"count": 3
}
],
"mem": {
"heap_used_in_bytes": 1671452984,
"heap_max_in_bytes": 3113877504
},
"threads": 197
},
"fs": {
"total_in_bytes": 126144983040,
"free_in_bytes": 115344986112,
"available_in_bytes": 115344986112
},
"plugins": [],
"network_types": {
"transport_types": {
"netty4": 3
},
"http_types": {
"netty4": 3
}
},
"discovery_types": {
"zen": 3
},
"packaging_types": [
{
"flavor": "default",
"type": "tar",
"count": 3
}
],
"ingest": {
"number_of_pipelines": 2,
"processor_stats": {
"gsub": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"script": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
}
}
}
}
}
Node stats
$ curl -q -s http://10.10.10.1:39200/_nodes
$ curl -q -s http://10.10.10.1:39200/_nodes/usage
$ curl -q -s http://10.10.10.2:39200/_nodes/stats/fs?pretty
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "rally-benchmark",
"nodes" : {
"Zyv-MlSUQeOpU-__o2NblA" : {
"timestamp" : 1667082588000,
"name" : "rally-node-1",
"transport_address" : "10.10.10.2:39300",
"host" : "10.10.10.2",
"ip" : "10.10.10.2:39300",
"roles" : [
"data",
"data_cold",
"data_content",
"data_frozen",
"data_hot",
"data_warm",
"ingest",
"master",
"remote_cluster_client",
"transform"
],
"attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"fs" : {
"timestamp" : 1667082588001,
"total" : {
"total_in_bytes" : 42048327680,
"free_in_bytes" : 39852097536,
"available_in_bytes" : 39852097536
},
"data" : [
{
"path" : "/home/es/.rally/benchmarks/races/53e2ac0b-c648-4153-97d4-46749f90daed/rally-node-1/install/elasticsearch-7.17.0/data/nodes/0",
"mount" : "/home (/dev/mapper/centos-home)",
"type" : "xfs",
"total_in_bytes" : 42048327680,
"free_in_bytes" : 39852097536,
"available_in_bytes" : 39852097536
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "dm-2",
"operations" : 52115,
"read_operations" : 0,
"write_operations" : 52115,
"read_kilobytes" : 0,
"write_kilobytes" : 7926721,
"io_time_in_millis" : 8549
}
],
"total" : {
"operations" : 52115,
"read_operations" : 0,
"write_operations" : 52115,
"read_kilobytes" : 0,
"write_kilobytes" : 7926721,
"io_time_in_millis" : 8549
}
}
}
},
[..]
}
}
The data path can also be checked in node-config.json.
$ cat races/aa826112-d371-4f09-9b68-f9084e7c9e0b/node-config.json
{
"build-type": "tar",
"car-runtime-jdks": "17,16,15,14,13,12,11,8",
"car-provides-bundled-jdk": true,
"ip": "10.10.10.1",
"node-name": "rally-node-0",
"node-root-path": "/home/es/.rally/benchmarks/races/aa826112-d371-4f09-9b68-f9084e7c9e0b/rally-node-0",
"binary-path": "/home/es/.rally/benchmarks/races/aa826112-d371-4f09-9b68-f9084e7c9e0b/rally-node-0/install/elasticsearch-7.17.0",
"data-paths": [
"/home/es/.rally/benchmarks/races/aa826112-d371-4f09-9b68-f9084e7c9e0b/rally-node-0/install/elasticsearch-7.17.0/data"
]
}
Shards and Replicas
List all the available shards
$ curl -q -s http://10.10.10.1:39200/_cat/shards
.geoip_databases 0 p STARTED 41 39.1mb 10.10.10.2 rally-node-1
.geoip_databases 0 r STARTED 41 39.1mb 10.10.10.3 rally-node-2
geonames 4 p STARTED 2279907 610.1mb 10.10.10.1 rally-node-0
geonames 3 p STARTED 2282121 605.1mb 10.10.10.2 rally-node-1
geonames 2 p STARTED 2280777 603.3mb 10.10.10.3 rally-node-2
geonames 1 p STARTED 2277042 609.3mb 10.10.10.1 rally-node-0
geonames 0 p STARTED 2276656 605.3mb 10.10.10.2 rally-node-1
.ds-.logs-deprecation.elasticsearch-default-2022.10.29-000001 0 r STARTED 10.10.10.1 rally-node-0
.ds-.logs-deprecation.elasticsearch-default-2022.10.29-000001 0 p STARTED 10.10.10.3 rally-node-2
.ds-ilm-history-5-2022.10.29-000001 0 r STARTED 10.10.10.2 rally-node-1
.ds-ilm-history-5-2022.10.29-000001 0 p STARTED 10.10.10.1 rally-node-0 0 p STARTED 10.10.10.1 rally-node-0
List all the available shards for the target index
By default, there are 5 primary shards for the target index.
$ curl -q -s http://10.10.10.1:39200/_cat/shards/geonames
geonames 4 p STARTED 2279907 610.1mb 10.10.10.1 rally-node-0
geonames 3 p STARTED 2282121 605.1mb 10.10.10.2 rally-node-1
geonames 2 p STARTED 2280777 603.3mb 10.10.10.3 rally-node-2
geonames 1 p STARTED 2277042 609.3mb 10.10.10.1 rally-node-0
geonames 0 p STARTED 2276656 605.3mb 10.10.10.2 rally-node-1
Check the settings of target index
By default, there are 5 primary shards and no replica shards for the target index.
$ curl -q -s http://10.10.10.1:39200/geonames/_settings | jq -r
{
"geonames": {
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_content"
}
}
},
"number_of_shards": "5",
"provided_name": "geonames",
"creation_date": "1667016351070",
"requests": {
"cache": {
"enable": "false"
}
},
"store": {
"type": "fs"
},
"number_of_replicas": "0",
"uuid": "ecaI9RlcSvqu0Lwo4f3nOA",
"version": {
"created": "7170099"
}
}
}
}
}
Change number_of_replicas for the target index
$ curl -X PUT -H "Content-Type: application/json" "http://10.10.10.1:39200/geonames/_settings" -d '{"number_of_replicas":1}'
{"acknowledged":true}
$ curl -q -s http://10.10.10.1:39200/geonames/_settings | jq -r
{
"geonames": {
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_content"
}
}
},
"number_of_shards": "5",
"provided_name": "geonames",
"creation_date": "1667016351070",
"requests": {
"cache": {
"enable": "false"
}
},
"store": {
"type": "fs"
},
"number_of_replicas": "1",
"uuid": "ecaI9RlcSvqu0Lwo4f3nOA",
"version": {
"created": "7170099"
}
}
}
}
}
Verify the index replicas
After changing number_of_replicas from 0 to 1, one replica shard is created for each primary shard. The primary shard and replica shard are located on different nodes. If a node goes down, the replica shards will automatically become primary shards and the cluster still works properly.
$ curl -q -s http://10.10.10.1:39200/_cat/shards/geonames
geonames 3 p STARTED 2282121 605.1mb 10.10.10.2 rally-node-1
geonames 3 r STARTED 2282121 605.1mb 10.10.10.1 rally-node-0
geonames 4 r STARTED 2279907 610.1mb 10.10.10.2 rally-node-1
geonames 4 p STARTED 2279907 610.1mb 10.10.10.1 rally-node-0
geonames 2 r STARTED 2280777 603.3mb 10.10.10.2 rally-node-1
geonames 2 p STARTED 2280777 603.3mb 10.10.10.3 rally-node-2
geonames 1 p STARTED 2277042 609.3mb 10.10.10.1 rally-node-0
geonames 1 r STARTED 2277042 609.3mb 10.10.10.3 rally-node-2
geonames 0 p STARTED 2276656 605.3mb 10.10.10.2 rally-node-1
geonames 0 r STARTED 2276656 605.3mb 10.10.10.3 rally-node-2
Task management
$ curl -q -s http://10.10.10.1:39200/_tasks
{
"nodes": {
"Zyv-MlSUQeOpU-__o2NblA": {
"name": "rally-node-1",
"transport_address": "10.10.10.2:39300",
"host": "10.10.10.2",
"ip": "10.10.10.2:39300",
"roles": [
"data",
"data_cold",
"data_content",
"data_frozen",
"data_hot",
"data_warm",
"ingest",
"master",
"remote_cluster_client",
"transform"
],
"attributes": {
"xpack.installed": "true",
"transform.node": "true"
},
"tasks": {
"Zyv-MlSUQeOpU-__o2NblA:500850": {
"node": "Zyv-MlSUQeOpU-__o2NblA",
"id": 500850,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1667176733761,
"running_time_in_nanos": 125953,
"cancellable": false,
"parent_task_id": "viZm_2hlQbib6qqccdXgNA:324323",
"headers": {}
}
}
},
[..]
}
}