The current script allows me to scrape only a single page but i would like to scarpe all 5 pages from the source url. How can i loop/iterate through remaining 4 pages?
#Import Libraries
from bs4 import BeautifulSoup
import requests
import csv
source = requests.get('https://www.sustainalytics.com/esg-ratings/?industry=Aerospace%20&%20Defense¤tpage=1').text
soup = BeautifulSoup(source, 'lxml')
#Start CSV
csv_file = open('aerospacedata_1.csv', 'w')
csv_writer = csv.writer(csv_file)
csv_writer.writerow(['company_name', 'company_exchange', 'company_risk'])
#Scrape Data from Web and write to csv
for company_info in soup.find_all(class_='company-row d-flex'):
company_name = company_info.a.text
company_exchange = company_info.find("small").text
company_risk = company_info.find("div", class_="col-2").text
print(company_name, company_exchange,company_risk)
csv_writer.writerow([company_name, company_exchange, company_risk])
csv_file.close()
Output:
company_name company_exchange company_risk
AECC Aviation Power Co Ltd SHG:600893 53.3
Airbus SE PAR:AIR 30.3
Aselsan Elektronik Sanayi ve Ticaret AS IST:ASELS 31.6
AVIC Aircraft Co., Ltd. SHE:000768 54.4
AVIC Shenyang Aircraft Co. Ltd. SHG:600760 51.3
AviChina Industry & Technology Company Limited HKG:2357 45.2
BAE Systems PLC LON:BA 34.3
Bombardier Inc. TSE:BBD.B 30
BWX Technologies, Inc. NYS:BWXT 42.3
CAE Inc. TSE:CAE 32.4