0

I'm looking to use the Excel Power Query to import some json that looks like the following (but much bigger, more fields etc.):

example-records.json

{
    "records": {
        "record_id_1": {
            "file_no": "5792C",
            "loads": {
                "load_id_1": {
                    "docket_no": "3116115"
                },
                "load_id_2": {
                    "docket_no": "3116118"
                },
                "load_id_3": {
                    "docket_no": "3208776"
                }
            }
        },
        "record_id_2": {
            "file_no": "5645C",
            "loads": {
                "load_id_4": {
                    "docket_no": "2000527155"
                },
                "load_id_5": {
                    "docket_no": "2000527156"
                },
                "load_id_6": {
                    "docket_no": "2000527146"
                }
            }
        }
    }
}

I want to get a table like the following at the load_id / docket level. A row per load_id

enter image description here

What I've tried

Clicking buttons in power query UI I get the following.

The problem is I can't include a file_no column and this doesn't work when there are lots of load ids.

let
    Source = Json.Document(File.Contents("H:\Software\Site Apps\example-records.json")),
    records = Source[records],
    #"Converted to Table" = Record.ToTable(records),
    #"Expanded Value" = Table.ExpandRecordColumn(#"Converted to Table", "Value", {"file_no", "loads"}, {"Value.file_no", "Value.loads"}),
    #"Removed Columns" = Table.RemoveColumns(#"Expanded Value",{"Value.file_no"}),
    #"Expanded Value.loads" = Table.ExpandRecordColumn(#"Removed Columns", "Value.loads", {"load_id_1", "load_id_2", "load_id_3", "load_id_4", "load_id_5", "load_id_6"}, {"Value.loads.load_id_1", "Value.loads.load_id_2", "Value.loads.load_id_3", "Value.loads.load_id_4", "Value.loads.load_id_5", "Value.loads.load_id_6"}),
    #"Unpivoted Columns" = Table.UnpivotOtherColumns(#"Expanded Value.loads", {"Name"}, "Attribute", "Value"),
    #"Expanded Value1" = Table.ExpandRecordColumn(#"Unpivoted Columns", "Value", {"docket_no"}, {"Value.docket_no"})
in
    #"Expanded Value1"

enter image description here

2 Answers 2

1

You can use

let Source = JSON(Json.Document(File.Contents("c:\temp\example.json"))),
#"Removed Other Columns" = Table.SelectColumns(Source,{"Name.1", "Name.3", "Value"}),
#"Added Custom" = Table.AddColumn(#"Removed Other Columns", "Custom", each if [Name.3]=null then [Value] else null),
#"Filled Down" = Table.FillDown(#"Added Custom",{"Custom"}),
#"Filtered Rows" = Table.SelectRows(#"Filled Down", each ([Name.3] <> null))
in  #"Filtered Rows"

based on this function I named JSON which comes from Imke https://www.thebiccountant.com/2018/06/17/automatically-expand-all-fields-from-a-json-document-in-power-bi-and-power-query/ which is reproduced below

let
func = (JSON) =>
    let
        Source = JSON, 
        ParseJSON = try Json.Document(Source) otherwise Source, 
        TransformForTable = 
            if Value.Is(ParseJSON, type record) then
                Record.ToTable(ParseJSON)
            else
                #table(
                    {"Name", "Value"}, 
                    List.Zip({List.Repeat({0}, List.Count(ParseJSON)), ParseJSON})
                ), 
        AddSort = Table.Buffer(Table.AddColumn(TransformForTable, "Sort", each 0)), 
        LG = List.Skip(
            List.Generate(
                () => [Next = AddSort, Counter = 1, AddIndex = #table({"Sort"}, {{""}})], 
                each [AddIndex]{0}[Sort] <> "End", 
                each [
                    AddIndex = Table.AddIndexColumn([Next], "Index", 0, 1), 
                    MergeSort = Table.CombineColumns(
                        Table.TransformColumnTypes(
                            AddIndex, 
                            {{"Sort", type text}, {"Index", type text}}, 
                            "en-GB"
                        ), 
                        {"Sort", "Index"}, 
                        Combiner.CombineTextByDelimiter(".", QuoteStyle.None), 
                        "Sort"
                    ), 
                    PJson = Table.TransformColumns(
                        MergeSort, 
                        {{"Value", each try Json.Document(_) otherwise _}}
                    ), 
                    AddType = Table.AddColumn(
                        PJson, 
                        "Type", 
                        each 
                            if Value.Is([Value], type record) then
                                "Record"
                            else if Value.Is([Value], type list) then
                                "List"
                            else if Value.Is([Value], type table) then
                                "Table"
                            else
                                "other"
                    ), 
                    AddStatus = Table.AddColumn(
                        AddType, 
                        "Status", 
                        each if [Type] = "other" then "Finished" else "Unfinished"
                    ), 
                    Finished = Table.SelectRows(AddStatus, each ([Status] = "Finished")), 
                    Unfinished = Table.SelectRows(AddStatus, each ([Status] = "Unfinished")), 
                    AddNext = Table.AddColumn(
                        Unfinished, 
                        "Next", 
                        each if [Type] = "Record" then {[Value]} else [Value]
                    ), 
                    RemoveCols = Table.RemoveColumns(AddNext, {"Value", "Type", "Status"}), 
                    ExpandNext = Table.ExpandListColumn(RemoveCols, "Next"), 
                    AddIndex2 = Table.AddIndexColumn(ExpandNext, "Index", 0, 1), 
                    MergeSort2 = Table.CombineColumns(
                        Table.TransformColumnTypes(
                            AddIndex2, 
                            {{"Sort", type text}, {"Index", type text}}, 
                            "en-GB"
                        ), 
                        {"Sort", "Index"}, 
                        Combiner.CombineTextByDelimiter(".", QuoteStyle.None), 
                        "Sort"
                    ), 
                    TransformRecord = Table.TransformColumns(
                        MergeSort2, 
                        {
                            {
                                "Next", 
                                each try
                                    Record.ToTable(_)
                                otherwise
                                    try
                                        
                                            if Value.Is(Text.From(_), type text) then
                                                #table({"Value"}, {{_}})
                                            else
                                                _
                                    otherwise
                                        _
                            }
                        }
                    ), 
                    FilterOutNulls = Table.SelectRows(TransformRecord, each [Next] <> null), 
                    Next = 
                        if Table.IsEmpty(FilterOutNulls) then
                            #table({"Sort"}, {{"End"}})
                        else if Value.Is(FilterOutNulls[Next]{0}, type table) = true then
                            Table.ExpandTableColumn(
                                FilterOutNulls, 
                                "Next", 
                                {"Name", "Value"}, 
                                {"Name." & Text.From([Counter]), "Value"}
                            )
                        else
                            Table.RenameColumns(FilterOutNulls, {{"Next", "Value"}}), 
                    Counter = [Counter] + 1
                ], 
                each Table.AddColumn([Finished], "Level", (x) => _[Counter] - 2)
            )
        ), 
        Check = LG{2}, 
        Combine = Table.Combine(LG), 
        Clean = Table.RemoveColumns(Combine, {"Status", "Type"}), 
        Trim = Table.TransformColumns(Clean, {{"Sort", each Text.Trim(_, "."), type text}}), 
        // Dynamic Padding for the sort-column so that it sorts by number in text strings
        SelectSort = Table.SelectColumns(Trim, {"Sort"}), 
        SplitSort = Table.AddColumn(
            SelectSort, 
            "Custom", 
            each List.Transform(try Text.Split([Sort], ".") otherwise {}, Number.From)
        ), 
        ToTable = Table.AddColumn(
            SplitSort, 
            "Splitted", 
            each Table.AddIndexColumn(Table.FromColumns({[Custom]}), "Pos", 1, 1)
        ), 
        ExpandTable = Table.ExpandTableColumn(ToTable, "Splitted", {"Column1", "Pos"}), 
        GroupPos = Table.Group(
            ExpandTable, 
            {"Pos"}, 
            {{"All", each _, type table}, {"Max", each List.Max([Column1]), type text}}
        ), 
        Digits = Table.AddColumn(GroupPos, "Digits", each Text.Length(Text.From([Max]))), 
        FilteredDigits = List.Buffer(Table.SelectRows(Digits, each ([Digits] <> null))[Digits]), 
        SortNew = Table.AddColumn(
            Trim, 
            "SortBy", 
            each Text.Combine(
                List.Transform(
                    List.Zip({Text.Split([Sort], "."), List.Positions(Text.Split([Sort], "."))}), 
                    each Text.PadStart(_{0}, FilteredDigits{_{1}}, "0")
                ), 
                "."
            )
        ), 
        FilterNotNull = Table.SelectRows(SortNew, each ([Value] <> null)), 
        Reorder = Table.ReorderColumns(
            FilterNotNull, 
            {"Value", "Level", "Sort", "SortBy"}
                & List.Difference(
                    Table.ColumnNames(FilterNotNull), 
                    {"Value", "Level", "Sort", "SortBy"}
                )
        ), 
        Dots = Table.AddColumn(
            #"Reorder", 
            "Dots", 
            each List.Select(Table.ColumnNames(#"Reorder"), (l) => Text.StartsWith(l, "Name"))
        ), 
        // This sort is just to view in the query editor. When loaded to the data model it will not be kept. Use "Sort by column" in the data model instead.
        Sort = Table.Sort(Dots, {{"SortBy", Order.Ascending}})
    in
        Sort, 
documentation = [
    Documentation.Name = " Table.JsonExpandAll ", 
    Documentation.Description
        = " Dynamically expands the <Json> Record and returns values in one column and additional columns to navigate. ", 
    Documentation.LongDescription
        = " Dynamically expands the <Json> Record and returns values in one column and additional columns to navigate. Input can be JSON in binary format or the already parsed JSON. ", 
    Documentation.Category = " Table ", 
    Documentation.Version = " 1.2: Added column [Dots] (22/02/2019)", 
    Documentation.Author = " Imke Feldmann: www.TheBIccountant.com . ", 
    Documentation.Examples = {[Description = "  ", Code = "  ", Result = "  "]}
]
in
Value.ReplaceType(func, Value.ReplaceMetadata(Value.Type(func), documentation))

enter image description here

Sign up to request clarification or add additional context in comments.

2 Comments

I get Expression.Error: We cannot convert a value of type Record to type Table.
I am using your data pasted in question as the source json file. With my code it works to produce the output from my image without error. Does it work for you with your own sample data? Does it then not work with the real data?
1

Managed to use an added custom column, the action that enables the expansion to one load id per row.

    #"Added Custom" = Table.AddColumn(#"Expanded Value", "Custom", each Record.ToTable([Value.loads]))
let
    Source = Json.Document(File.Contents("H:\Software\Site Apps\example-records.json")),
    records = Source[records],
    #"Converted to Table" = Record.ToTable(records),
    #"Expanded Value" = Table.ExpandRecordColumn(#"Converted to Table", "Value", {"file_no", "loads"}, {"Value.file_no", "Value.loads"}),
    #"Added Custom" = Table.AddColumn(#"Expanded Value", "Custom", each Record.ToTable([Value.loads])),
    #"Removed Columns" = Table.RemoveColumns(#"Added Custom",{"Value.loads"}),
    #"Expanded Custom" = Table.ExpandTableColumn(#"Removed Columns", "Custom", {"Name", "Value"}, {"Custom.Name", "Custom.Value"}),
    #"Expanded Custom.Value" = Table.ExpandRecordColumn(#"Expanded Custom", "Custom.Value", {"docket_no"}, {"Custom.Value.docket_no"}),
    #"Renamed Columns" = Table.RenameColumns(#"Expanded Custom.Value",{{"Name", "record_id"}, {"Value.file_no", "file_no"}, {"Custom.Name", "load_id"}, {"Custom.Value.docket_no", "docket_no"}})
in
    #"Renamed Columns"

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.