Perform multiple indexing operations using single API call to Elasticsearch cluster.
Create simple mapping template (date and message).
$ curl -X PUT -u elastic:elastic "https://192.168.8.153:9200/_component_template/nginx-logs-mappings?pretty" \
-H 'Content-Type: application/json' -d'
{
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"message": {
"type": "wildcard"
}
}
}
}
}'
{
"acknowledged" : true
}
Create basic settings template.
$ curl -X PUT -u elastic:elastic "https://192.168.8.153:9200/_component_template/nginx-logs-settings?pretty" \
-H 'Content-Type: application/json' -d'
{
"template": {
"settings": {
"index.lifecycle.name": "365-days-defaul",
"number_of_replicas": 1
}
}
}'
{
"acknowledged" : true
}
Create index template.
$ curl -X PUT -u elastic:elastic "https://192.168.8.153:9200/_index_template/nginx-logs-template?pretty" \
-H 'Content-Type: application/json' -d'
{
"index_patterns": ["nginx-logs*"],
"data_stream": { },
"composed_of": [ "nginx-logs-mappings", "nginx-logs-settings" ],
"priority": 500
}'
{
"acknowledged" : true
}
Create data stream.
$ curl -X PUT -u elastic:elastic "https://192.168.8.153:9200/_data_stream/nginx-logs?pretty"
{
"acknowledged" : true
}
Use bulk API to perform multiple indexing operations.
$ curl -X PUT -u elastic:elastic "https://192.168.8.153:9200/nginx-logs/_bulk?pretty" \
-H 'Content-Type: application/json' -d '
{ "create":{ } }
{ "@timestamp": "2023-07-25T12:37:41.000Z", "message": "172.16.0.203 - - [25/Jul/2023:12:37:41 +0000] \"GET / HTTP/1.0\" 200 13503 \"-\" \"Uptime-Kuma/1.21.3\"" }
{ "create":{ } }
{ "@timestamp": "2023-07-25T12:37:44.000Z", "message": "192.0.2.255 - - 172.16.0.122 - - [25/Jul/2023:12:37:44 +0000] \"GET /feed.xml HTTP/1.0\" 404 549 \"-\" \"-\"" }
{ "create":{ } }
{ "@timestamp": "2023-07-25T12:40:24.000ZZ", "message": "172.16.0.100 - - [25/Jul/2023:12:40:24 +0000] \"GET /index.xml HTTP/1.0\" 200 770768 \"-\" \"python-requests/2.31.0\"" }
{ "create":{ } }
{ "@timestamp": "2023-07-25T12:43:26.000Z", "message": "172.16.0.126 - - [25/Jul/2023:12:43:26 +0000] \"GET /index.xml HTTP/1.0\" 304 - \"-\" \"Feedly/1.0 (+http://www.feedly.com/fetcher.html; 117 subscribers; )\"" }
'
{
"took" : 132,
"errors" : false,
"items" : [
{
"create" : {
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "IMu_jIkB72uyfiJ8BwUh",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"_seq_no" : 0,
"_primary_term" : 1,
"status" : 201
}
},
{
"create" : {
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "Icu_jIkB72uyfiJ8BwUh",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"_seq_no" : 1,
"_primary_term" : 1,
"status" : 201
}
},
},
{
"create" : {
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "Isu_jIkB72uyfiJ8BwUh",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"_seq_no" : 2,
"_primary_term" : 1,
"status" : 201
}
},
{
"create" : {
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "I8u_jIkB72uyfiJ8BwUh",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"_seq_no" : 3,
"_primary_term" : 1,
"status" : 201
}
}
]
}
Inspect index details.
$ curl -s -u elastic:elastic "https://192.168.8.153:9200/_cat/indices/nginx-logs?v"
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open .ds-nginx-logs-2023.07.25-000001 b82UcNfBStaGOXsFcaphvw 1 1 4 0 11.9kb 5.9kb
Perform sample query.
curl --silent \
--header 'Content-Type: application/json' \
-u elastic:elastic "https://192.168.8.153:9200/nginx-logs/_search?pretty=true" \
--data-ascii \
'{
"query": {
"match_all": {}
}
}'
{
"took" : 628,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 4,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "IMu_jIkB72uyfiJ8BwUh",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2023-07-25T12:37:41.000Z",
"message" : "172.16.0.203 - - [25/Jul/2023:12:37:41 +0000] \"GET / HTTP/1.0\" 200 13503 \"-\" \"Uptime-Kuma/1.21.3\""
}
},
{
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "Icu_jIkB72uyfiJ8BwUh",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2023-07-25T12:37:44.000Z",
"message" : "192.0.2.255 - - 172.16.0.122 - - [25/Jul/2023:12:37:44 +0000] \"GET /feed.xml HTTP/1.0\" 404 549 \"-\" \"-\""
}
},
{
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "Isu_jIkB72uyfiJ8BwUh",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2023-07-25T12:40:24.000ZZ",
"message" : "172.16.0.100 - - [25/Jul/2023:12:40:24 +0000] \"GET /index.xml HTTP/1.0\" 200 770768 \"-\" \"python-requests/2.31.0\""
}
},
{
"_index" : ".ds-nginx-logs-2023.07.25-000001",
"_id" : "I8u_jIkB72uyfiJ8BwUh",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2023-07-25T12:43:26.000Z",
"message" : "172.16.0.126 - - [25/Jul/2023:12:43:26 +0000] \"GET /index.xml HTTP/1.0\" 304 - \"-\" \"Feedly/1.0 (+http://www.feedly.com/fetcher.html; 117 subscribers; )\""
}
}
]
}
}
Additional notes
