Categories
SysOps

How to automatically rollover Elasticsearch index

Automatically rollover Elasticsearch index using by time.

Elasticsearch version.

$ curl --insecure --user elastic:secret "https://172.17.0.4:9200/"
{
  "name" : "a67038f591a0",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "8A6Tz4hNTWu0o_QwGmOnqg",
  "version" : {
    "number" : "8.2.3",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "9905bfb62a3f0b044948376b4f607f70a8a151b4",
    "build_date" : "2022-06-08T22:21:36.455508792Z",
    "build_snapshot" : false,
    "lucene_version" : "9.1.0",
    "minimum_wire_compatibility_version" : "7.17.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "You Know, for Search"
}

Display lifecycle policies.

$ curl --silent --insecure -user elastic:secret 'https://172.17.0.4:9200/_ilm/policy?pretty' | jq --raw-output ' keys[]'
.deprecation-indexing-ilm-policy
.fleet-actions-results-ilm-policy
.monitoring-8-ilm-policy
180-days-default
30-days-default
365-days-default
7-days-default
90-days-default
ilm-history-ilm-policy
logs
metrics
ml-size-based-ilm-policy
slm-history-ilm-policy
synthetics
watch-history-ilm-policy-16

Display specific lifecycle policy.

$ curl --silent --insecure -user elastic:secret 'https://172.17.0.4:9200/_ilm/policy/365-days-default?pretty' 
{
  "365-days-default" : {
    "version" : 1,
    "modified_date" : "2022-06-23T21:01:55.454Z",
    "policy" : {
      "phases" : {
        "warm" : {
          "min_age" : "2d",
          "actions" : {
            "shrink" : {
              "number_of_shards" : 1
            },
            "forcemerge" : {
              "max_num_segments" : 1
            }
          }
        },
        "cold" : {
          "min_age" : "30d",
          "actions" : { }
        },
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_primary_shard_size" : "50gb",
              "max_age" : "30d"
            }
          }
        },
        "delete" : {
          "min_age" : "365d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      },
      "_meta" : {
        "managed" : true,
        "description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days"
      }
    },
    "in_use_by" : {
      "indices" : [ ],
      "data_streams" : [ ],
      "composable_templates" : [ ]
    }
  }
}

Create lifecycle policy to trigger rollover after 1 day since the index creation time and delete indices older than a year.

$ curl 'https://172.17.0.4:9200/_ilm/policy/simple-lifecycle-policy-365?pretty' \
     --insecure --user elastic:secret \
     --header 'Content-Type: application/json'  \
     --request PUT \
     --data-raw '
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
             "max_age": "1d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "delete": {
        "min_age": "365d",
        "actions": {
          "delete": {
            "delete_searchable_snapshot": true
          }
        }
      }
    }
  }
}'

{
“acknowledged” : true
}

Create index template using earlier created lifecycle policy.

$ curl "https://172.17.0.4:9200/_index_template/recipes_template?pretty" \
     --insecure --user elastic:secret \
     --header 'Content-Type: application/json'  \
     --request PUT \
     --data-raw '
{
  "index_patterns":["recipes-*"],
  "template": {
    "settings": {
      "index": {
        "lifecycle":{
          "name":"simple-lifecycle-policy-365",
          "rollover_alias": "recipes"
        }
      }
    }
  }
}'
{
  "acknowledged" : true
}

Bootstrap the initial time series index with an alias configured as the write index.

$ curl 'https://172.17.0.4:9200/recipes-000001?pretty' \
     --insecure --user elastic:secret \
     --header 'Content-Type: application/json'  \
     --request PUT \
     --data-raw '
{
  "aliases": {
    "recipes": {
      "is_write_index": true
    }
  }
}'
{
  "acknowledged" : true,
  "shards_acknowledged" : true,
  "index" : "recipes-000001"
}

Create sample data.

$ curl --silent \
       --insecure --user elastic:secret \
       --header 'Content-Type: application/json'  \
       --request POST \
       "https://172.17.0.4:9200/recipes/_doc?refresh=wait_for&pretty=true" \
       --data-raw '
{ 
   "name": "Recipe #1"
}'
{
  "_index" : "recipes-000001",
  "_id" : "KUKEkoEBtl9YDvIiiis7",
  "_version" : 1,
  "result" : "created",
  "_shards" : {
    "total" : 2,
    "successful" : 1,
    "failed" : 0
  },
  "_seq_no" : 0,
  "_primary_term" : 1
}

List indices.

$ curl --insecure --user elastic:secret "https://172.17.0.4:9200/_cat/indices/"
yellow open recipes-000001 z1_lBIfwQ3eUeUHsUiiwJA 1 1 1 0 3.9kb 3.9kb

Create a new index by hand.

$ curl --insecure --user elastic:secret -X POST "https://172.17.0.4:9200/recipes/_rollover"
{
  "acknowledged" : true,
  "shards_acknowledged" : true,
  "old_index" : "recipes-000001",
  "new_index" : "recipes-000002",
  "rolled_over" : true,
  "dry_run" : false,
  "conditions" : { }
}

List indices.

$ curl --insecure --user elastic:secret "https://172.17.0.4:9200/_cat/indices/"
yellow open recipes-000002 A_lFh3WDT2CXoGuqddRvtA 1 1 0 0  225b  225b
yellow open recipes-000001 z1_lBIfwQ3eUeUHsUiiwJA 1 1 1 0 3.9kb 3.9kb

Explain lifecycle status.

$ curl --insecure --user elastic:secret "https://172.17.0.4:9200/recipes-*/_ilm/explain?pretty"
{
  "indices" : {
    "recipes-000002" : {
      "index" : "recipes-000002",
      "managed" : true,
      "policy" : "simple-lifecycle-policy-365",
      "index_creation_date_millis" : 1656020689137,
      "time_since_index_creation" : "42.79s",
      "lifecycle_date_millis" : 1656020689137,
      "age" : "42.79s",
      "phase" : "hot",
      "phase_time_millis" : 1656020689225,
      "action" : "rollover",
      "action_time_millis" : 1656020689225,
      "step" : "check-rollover-ready",
      "step_time_millis" : 1656020689225,
      "phase_execution" : {
        "policy" : "simple-lifecycle-policy-365",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "set_priority" : {
              "priority" : 100
            },
            "rollover" : {
              "max_age" : "1d"
            }
          }
        },
        "version" : 1,
        "modified_date_in_millis" : 1656019887353
      }
    },
    "recipes-000001" : {
      "index" : "recipes-000001",
      "managed" : true,
      "policy" : "simple-lifecycle-policy-365",
      "index_creation_date_millis" : 1656020144560,
      "time_since_index_creation" : "9.78m",
      "lifecycle_date_millis" : 1656020144560,
      "age" : "9.78m",
      "phase" : "hot",
      "phase_time_millis" : 1656020144683,
      "action" : "rollover",
      "action_time_millis" : 1656020144683,
      "step" : "check-rollover-ready",
      "step_time_millis" : 1656020144683,
      "phase_execution" : {
        "policy" : "simple-lifecycle-policy-365",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "set_priority" : {
              "priority" : 100
            },
            "rollover" : {
              "max_age" : "1d"
            }
          }
        },
        "version" : 1,
        "modified_date_in_millis" : 1656019887353
      }
    }
  }
}

Index will rotate automatically.