Python Scrape specific JS data

Question

Im having some trouble extracting the following data from a page:

I have highlighted the json I would like to obtain from the page.

I have also pasted the javascript section it is in below:

<script type="text/x-magento-init">
            {
                "#conf-select-attr-173": {
                    "Magento_ConfigurableProduct/js/configurable/select/action": {
                        "config": {"attributes":{"173":{"id":"173","code":"Size","label":"Size","options":[{"id":"342","label":"Footwear-38","products":["104984"]},{"id":"345","label":"Footwear-39","products":["104985"]},{"id":"347","label":"Footwear-39.5","products":["104986"]},{"id":"349","label":"Footwear-40","products":["104987"]},{"id":"351","label":"Footwear-40.5","products":["104988"]},{"id":"354","label":"Footwear-41.5","products":["104989"]},{"id":"355","label":"Footwear-42","products":["104990"]},{"id":"357","label":"Footwear-42.5","products":["104991"]},{"id":"360","label":"Footwear-43.5","products":["104992"]},{"id":"361","label":"Footwear-44","products":["104993"]},{"id":"363","label":"Footwear-44.5","products":["104994"]},{"id":"364","label":"Footwear-45","products":["104995"]},{"id":"367","label":"Footwear-46","products":["104996"]},{"id":"369","label":"Footwear-46.5","products":["104997"]}],"position":"0"}},"template":"<%- data.price %>\u00a0 \u20ac","currencyFormat":"%s\u00a0 \u20ac","optionPrices":{"104984":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104985":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104986":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104987":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104988":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104989":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104990":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104991":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104992":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104993":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104994":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104995":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104996":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104997":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]}},"priceFormat":{"pattern":"%s\u00a0 \u20ac","precision":2,"requiredPrecision":2,"decimalSymbol":",","groupSymbol":".","groupLength":3,"integerRequired":1},"prices":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9}},"productId":"104998","chooseText":"Choose an Option...","images":[],"index":{"104984":{"173":"342"},"104985":{"173":"345"},"104986":{"173":"347"},"104987":{"173":"349"},"104988":{"173":"351"},"104989":{"173":"354"},"104990":{"173":"355"},"104991":{"173":"357"},"104992":{"173":"360"},"104993":{"173":"361"},"104994":{"173":"363"},"104995":{"173":"364"},"104996":{"173":"367"},"104997":{"173":"369"}},"sku":{"default":"1201A429-300","104984":"1201A429-300-Footwear-38","104985":"1201A429-300-Footwear-39","104986":"1201A429-300-Footwear-39.5","104987":"1201A429-300-Footwear-40","104988":"1201A429-300-Footwear-40.5","104989":"1201A429-300-Footwear-41.5","104990":"1201A429-300-Footwear-42","104991":"1201A429-300-Footwear-42.5","104992":"1201A429-300-Footwear-43.5","104993":"1201A429-300-Footwear-44","104994":"1201A429-300-Footwear-44.5","104995":"1201A429-300-Footwear-45","104996":"1201A429-300-Footwear-46","104997":"1201A429-300-Footwear-46.5"},"stock":{"104984":{"is_salable":true,"qty":1},"104985":{"is_salable":true,"qty":1},"104986":{"is_salable":true,"qty":0},"104987":{"is_salable":true,"qty":1},"104988":{"is_salable":true,"qty":1},"104989":{"is_salable":true,"qty":2},"104990":{"is_salable":true,"qty":0},"104991":{"is_salable":true,"qty":0},"104992":{"is_salable":true,"qty":3},"104993":{"is_salable":true,"qty":2},"104994":{"is_salable":true,"qty":1},"104995":{"is_salable":true,"qty":0},"104996":{"is_salable":true,"qty":0},"104997":{"is_salable":true,"qty":0}}},
                        "selected": ""
                    }
                }
            }
        </script>

How can I obtain this quickly and efficiently - I have tried using Bs4 but I always get an object value of None returned. Please could someone show me how this can be done :)

Thanks!

this script looks like JSON data - so use module json to convert it to Python dictionary (ie. data) and get what you want - data["#conf-select-attr-173"]["Magento_ConfigurableProduct/js/configurable/select/action"]["config"]. Eventually you may get 4-th line from this text, remove "config": and again use json to convert it to Python dictionary — furas
– furas, Commented Jun 14, 2021 at 8:45

furas · Accepted Answer · 2021-06-14 08:59:40Z

This script looks like JSON data - so use module json to convert it to Python dictionary (ie. data) and get what you want -

data["#conf-select-attr-173"]["Magento_ConfigurableProduct/js/configurable/select/action"]["config"]

html = '''<script type="text/x-magento-init">
            {
                "#conf-select-attr-173": {
                    "Magento_ConfigurableProduct/js/configurable/select/action": {
                        "config": {"attributes":{"173":{"id":"173","code":"Size","label":"Size","options":[{"id":"342","label":"Footwear-38","products":["104984"]},{"id":"345","label":"Footwear-39","products":["104985"]},{"id":"347","label":"Footwear-39.5","products":["104986"]},{"id":"349","label":"Footwear-40","products":["104987"]},{"id":"351","label":"Footwear-40.5","products":["104988"]},{"id":"354","label":"Footwear-41.5","products":["104989"]},{"id":"355","label":"Footwear-42","products":["104990"]},{"id":"357","label":"Footwear-42.5","products":["104991"]},{"id":"360","label":"Footwear-43.5","products":["104992"]},{"id":"361","label":"Footwear-44","products":["104993"]},{"id":"363","label":"Footwear-44.5","products":["104994"]},{"id":"364","label":"Footwear-45","products":["104995"]},{"id":"367","label":"Footwear-46","products":["104996"]},{"id":"369","label":"Footwear-46.5","products":["104997"]}],"position":"0"}},"template":"<%- data.price %>\u00a0 \u20ac","currencyFormat":"%s\u00a0 \u20ac","optionPrices":{"104984":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104985":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104986":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104987":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104988":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104989":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104990":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104991":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104992":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104993":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104994":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104995":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104996":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104997":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]}},"priceFormat":{"pattern":"%s\u00a0 \u20ac","precision":2,"requiredPrecision":2,"decimalSymbol":",","groupSymbol":".","groupLength":3,"integerRequired":1},"prices":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9}},"productId":"104998","chooseText":"Choose an Option...","images":[],"index":{"104984":{"173":"342"},"104985":{"173":"345"},"104986":{"173":"347"},"104987":{"173":"349"},"104988":{"173":"351"},"104989":{"173":"354"},"104990":{"173":"355"},"104991":{"173":"357"},"104992":{"173":"360"},"104993":{"173":"361"},"104994":{"173":"363"},"104995":{"173":"364"},"104996":{"173":"367"},"104997":{"173":"369"}},"sku":{"default":"1201A429-300","104984":"1201A429-300-Footwear-38","104985":"1201A429-300-Footwear-39","104986":"1201A429-300-Footwear-39.5","104987":"1201A429-300-Footwear-40","104988":"1201A429-300-Footwear-40.5","104989":"1201A429-300-Footwear-41.5","104990":"1201A429-300-Footwear-42","104991":"1201A429-300-Footwear-42.5","104992":"1201A429-300-Footwear-43.5","104993":"1201A429-300-Footwear-44","104994":"1201A429-300-Footwear-44.5","104995":"1201A429-300-Footwear-45","104996":"1201A429-300-Footwear-46","104997":"1201A429-300-Footwear-46.5"},"stock":{"104984":{"is_salable":true,"qty":1},"104985":{"is_salable":true,"qty":1},"104986":{"is_salable":true,"qty":0},"104987":{"is_salable":true,"qty":1},"104988":{"is_salable":true,"qty":1},"104989":{"is_salable":true,"qty":2},"104990":{"is_salable":true,"qty":0},"104991":{"is_salable":true,"qty":0},"104992":{"is_salable":true,"qty":3},"104993":{"is_salable":true,"qty":2},"104994":{"is_salable":true,"qty":1},"104995":{"is_salable":true,"qty":0},"104996":{"is_salable":true,"qty":0},"104997":{"is_salable":true,"qty":0}}},
                        "selected": ""
                    }
                }
            }
        </script>'''

from bs4 import BeautifulSoup
import json

soup = BeautifulSoup(html, 'html.parser')
text = soup.find('script').string
#print(text)

data = json.loads(text)
config = data["#conf-select-attr-173"]["Magento_ConfigurableProduct/js/configurable/select/action"]["config"]

print(config)

Eventually you may get 4-th line from this text, remove "config": and , at the end and again use json to convert it to Python dictionary

html = '''<script type="text/x-magento-init">
            {
                "#conf-select-attr-173": {
                    "Magento_ConfigurableProduct/js/configurable/select/action": {
                        "config": {"attributes":{"173":{"id":"173","code":"Size","label":"Size","options":[{"id":"342","label":"Footwear-38","products":["104984"]},{"id":"345","label":"Footwear-39","products":["104985"]},{"id":"347","label":"Footwear-39.5","products":["104986"]},{"id":"349","label":"Footwear-40","products":["104987"]},{"id":"351","label":"Footwear-40.5","products":["104988"]},{"id":"354","label":"Footwear-41.5","products":["104989"]},{"id":"355","label":"Footwear-42","products":["104990"]},{"id":"357","label":"Footwear-42.5","products":["104991"]},{"id":"360","label":"Footwear-43.5","products":["104992"]},{"id":"361","label":"Footwear-44","products":["104993"]},{"id":"363","label":"Footwear-44.5","products":["104994"]},{"id":"364","label":"Footwear-45","products":["104995"]},{"id":"367","label":"Footwear-46","products":["104996"]},{"id":"369","label":"Footwear-46.5","products":["104997"]}],"position":"0"}},"template":"<%- data.price %>\u00a0 \u20ac","currencyFormat":"%s\u00a0 \u20ac","optionPrices":{"104984":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104985":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104986":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104987":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104988":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104989":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104990":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104991":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104992":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104993":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104994":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104995":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104996":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]},"104997":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9},"tierPrices":[]}},"priceFormat":{"pattern":"%s\u00a0 \u20ac","precision":2,"requiredPrecision":2,"decimalSymbol":",","groupSymbol":".","groupLength":3,"integerRequired":1},"prices":{"oldPrice":{"amount":129.9},"basePrice":{"amount":109.15966286555},"finalPrice":{"amount":129.9}},"productId":"104998","chooseText":"Choose an Option...","images":[],"index":{"104984":{"173":"342"},"104985":{"173":"345"},"104986":{"173":"347"},"104987":{"173":"349"},"104988":{"173":"351"},"104989":{"173":"354"},"104990":{"173":"355"},"104991":{"173":"357"},"104992":{"173":"360"},"104993":{"173":"361"},"104994":{"173":"363"},"104995":{"173":"364"},"104996":{"173":"367"},"104997":{"173":"369"}},"sku":{"default":"1201A429-300","104984":"1201A429-300-Footwear-38","104985":"1201A429-300-Footwear-39","104986":"1201A429-300-Footwear-39.5","104987":"1201A429-300-Footwear-40","104988":"1201A429-300-Footwear-40.5","104989":"1201A429-300-Footwear-41.5","104990":"1201A429-300-Footwear-42","104991":"1201A429-300-Footwear-42.5","104992":"1201A429-300-Footwear-43.5","104993":"1201A429-300-Footwear-44","104994":"1201A429-300-Footwear-44.5","104995":"1201A429-300-Footwear-45","104996":"1201A429-300-Footwear-46","104997":"1201A429-300-Footwear-46.5"},"stock":{"104984":{"is_salable":true,"qty":1},"104985":{"is_salable":true,"qty":1},"104986":{"is_salable":true,"qty":0},"104987":{"is_salable":true,"qty":1},"104988":{"is_salable":true,"qty":1},"104989":{"is_salable":true,"qty":2},"104990":{"is_salable":true,"qty":0},"104991":{"is_salable":true,"qty":0},"104992":{"is_salable":true,"qty":3},"104993":{"is_salable":true,"qty":2},"104994":{"is_salable":true,"qty":1},"104995":{"is_salable":true,"qty":0},"104996":{"is_salable":true,"qty":0},"104997":{"is_salable":true,"qty":0}}},
                        "selected": ""
                    }
                }
            }
        </script>'''

from bs4 import BeautifulSoup
import json

soup = BeautifulSoup(html, 'html.parser')
text = soup.find('script').string

lines = text.split('\n')

line4 = lines[4].strip()
line4 = line4.replace('"config": ', '')
line4 = line4[:-1]  #  remove `,` at the end

config = json.loads(line4)

print(config)

Collectives™ on Stack Overflow

Python Scrape specific JS data

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related