How can I get distinct values of nested fields in elasticsearch?

Question

I have below document structure in elasticsearch:

   root
|-- userid: string (nullable = true)
|-- name: string (nullable = true)
|-- applications: array (nullable = true)
|    |-- element: struct (containsNull = true)
|    |    |-- applicationid: string (nullable = true)
|    |    |-- createdat: string (nullable = true)
|    |    |-- source_name: string (nullable = true)
|    |    |-- accounts: array (nullable = true)
|    |    |    |-- element: struct (containsNull = true)
|    |    |    |    |-- applicationcreditreportaccountid: string 
                                                   (nullable = true)
|    |    |    |    |-- account_type: integer (nullable = true)
|    |    |    |    |-- account_department: string (nullable = true)

Below is the mapping of my index:

{
  "bureau_data" : {
    "mappings" : {
      "dynamic_date_formats" : [
        "yyyy-MM-dd"
      ],
      "dynamic_templates" : [
        {
          "objects" : {
            "match_mapping_type" : "object",
            "mapping" : {
              "type" : "nested"
            }
          }
        }
      ],
      "properties" : {
        "raw_derived" : {
          "type" : "nested",
          "properties" : {
            "applications" : {
              "type" : "nested",
              "properties" : {
                "accounts" : {
                  "type" : "nested",
                  "properties" : {
                    "account_type_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "accounttypeid" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "applicationcreditreportaccountid" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "currentbalance" : {
                      "type" : "long"
                    },
                    "dayspastdue" : {
                      "type" : "long"
                    },
                    "institution_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "institutionid" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    }
                  }
                },
                "applicationcreditreportid" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "applicationid" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "createdat" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "creditbureautypeid" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "dateofbirth" : {
                  "type" : "date",
                  "format" : "yyyy-MM-dd"
                },
                "firstname" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "lastname" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "score" : {
                  "type" : "long"
                },
                "source_name" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "status" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "updatedat" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "dob" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "firstname" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "lastname" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "middlename" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "mobilephone" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "source" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "userid" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          },
          "fielddata" : true
        }
      }
    }
  }
}

I want distinct values of account_type field which is a nested fields. I have tried query which is giving me only distinct count.

GET /my_index/_search?size=0
{
  "aggs": {
    "nested_path": {
      "nested": {
        "path": "raw_derived.applications.accounts"
      },
      "aggs": {
        "distinct_values": {
          "cardinality": {
            "field": "raw_derived.applications.accounts.account_type.keyword"
          }
        }
      }
    }
  }
}

I expected the output to have distinct values of account_type but the output is count only. Below is my output snippet:

  "hits" : {
    "total" : {
      "value" : 50,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "nested_path" : {
      "doc_count" : 828,
      "distinct_values" : {
        "value" : 70
      }
    }
  }
}

Below is the query I have tried and its working:

GET /bureau_data/_search?size=0
{
 "_source": "{aggregations}", 
 "aggs": {
   "unique": {
     "nested": {
       "path": "raw_derived.applications"
     },
         "aggs": {
           "score_unq": {
             "terms": {
               "field": "raw_derived.applications.source_name.keyword"
             }
           }
         }
       }
     }
   }

Any suggestion would be helpful

It would be good if you add the elastic mapping. GET index_name/_mapping instead of the structure :) — Richa
– Richa, Commented Sep 5, 2019 at 9:51

soumitra goswami · Accepted Answer · 2019-09-05 10:08:39Z

2

From the official documentation - Cardinality Aggregation :- A single-value metrics aggregation that calculates an approximate count of distinct values. Values can be extracted either from specific fields in the document or generated by a script.

Instead of aggregating by "cardinality" , try a terms aggregation as below:

{
"size":0,
"aggregations": {
    "distinct_values": {
      "terms": {
        "field": "raw_derived.applications.accounts.account_type.keyword",
        "size": 1000,
        "min_doc_count": 1,
        "order": [
          {
            "_count": "desc"
          },
          {
            "_key": "asc"
          }
        ]
      }
    }
}

answered Sep 5, 2019 at 10:08

soumitra goswami

8911 gold badge12 silver badges36 bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

adarsh2109 Over a year ago

Yeah I Just tried with terms but I have to give nested path first I guess... its working.Thanks a lot for your input. I have updated my query

Collectives™ on Stack Overflow

How can I get distinct values of nested fields in elasticsearch?

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related