I am trying to scrape the Trading Pools (Futures-Grid) table in the following link: https://www.binance.com/en/trading-bots using Selenium into a pandas dataframe. You can see the table when you scroll down a bit.
I was able to get the dataframe on the website on default view thanks to this post
However, I wanted to also click on the filter drop-down options available and select less than 1 day(≤1 Day) option from the Running Timesection
What I have attempted:
import pandas as pd
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
pd.set_option("display.max_rows", 100)
pd.set_option("display.max_columns", 100)
pd.set_option("display.width", 100)
url = "https://www.binance.com/en/trading-bots"
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-extensions')
browser = webdriver.Chrome(options=options)
browser.get(url)
# Click on the "Futures-Grid" button
futures_grid_button = WebDriverWait(browser, 30).until(
EC.element_to_be_clickable((By.ID, 'tab-futuresGrid'))
)
browser.execute_script("arguments[0].click();", futures_grid_button)
print("Clicked on the Futures-Grid button")
# Locate the Running Time dropdown
running_time_dropdown = WebDriverWait(browser, 30).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, 'div[name="runningTime"]'))
)
print("Running Time dropdown found")
# Force click the Running Time dropdown
browser.execute_script("arguments[0].click();", running_time_dropdown)
# Wait for the dropdown to be visible
dropdown_visible = WebDriverWait(browser, 30).until(
EC.visibility_of_element_located((By.CSS_SELECTOR, 'div.bn-sdd-dropdown.showing'))
)
print("Dropdown is visible")
# Locate the select element inside the dropdown
select_element = browser.find_element(By.CSS_SELECTOR, 'ul.bn-sdd-list')
# Create a Select object to interact with the dropdown
select = Select(select_element)
# Force click the "≤1 Day" option
one_day_option = select_element.find_element(By.XPATH, "//li[normalize-space()='≤1 Day']")
browser.execute_script("arguments[0].click();", one_day_option)
# Wait for the table rows to load
rows_element = WebDriverWait(browser, 30).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "div.body-row"))
)
print("Found {} rows".format(len(rows_element)))
headers = [
"Rank/Market",
"Direction",
"Running Time",
"ROI",
"PNL(USD)",
"Copy Popularity",
"Action",
]
rows = []
for row_element in rows_element:
cell_data = [i.text.replace('\n', ',') for i in row_element.find_elements(By.CSS_SELECTOR, 'div[class^="body-cell body-cell-"]')]
# print(cell_data)
rows.append(cell_data)
binance_bots_df = pd.DataFrame(rows, columns=headers)
print(binance_bots_df)
browser.quit()
But I get this error while attempting to filter using the dropdown menu:
Clicked on the Futures-Grid button
Running Time dropdown found
Output exceeds the size limit. Open the full output data in a text editor---------------------------------------------------------------------------
TimeoutException Traceback (most recent call last)
Cell In[28], line 42
39 browser.execute_script("arguments[0].click();", running_time_dropdown)
41 # Wait for the dropdown to be visible
---> 42 dropdown_visible = WebDriverWait(browser, 30).until(
43 EC.visibility_of_element_located((By.CSS_SELECTOR, 'div.bn-sdd-dropdown.showing'))
44 )
45 print("Dropdown is visible")
47 # Locate the select element inside the dropdown
File ~/anaconda3/envs/trading/lib/python3.10/site-packages/selenium/webdriver/support/wait.py:95, in WebDriverWait.until(self, method, message)
93 if time.monotonic() > end_time:
94 break
---> 95 raise TimeoutException(message, screen, stacktrace)
TimeoutException: Message:
Stacktrace:
0 chromedriver 0x0000000104d89670 chromedriver + 4298352
1 chromedriver 0x0000000104d81bbc chromedriver + 4266940
2 chromedriver 0x00000001049b4758 chromedriver + 280408
3 chromedriver 0x00000001049efb38 chromedriver + 523064
4 chromedriver 0x0000000104a28080 chromedriver + 753792
5 chromedriver 0x00000001049e22d0 chromedriver + 467664
6 chromedriver 0x00000001049e3354 chromedriver + 471892
...
13 chromedriver 0x0000000104d6d3b4 chromedriver + 4182964
14 chromedriver 0x0000000104d7c0f4 chromedriver + 4243700
15 libsystem_pthread.dylib 0x0000000199cebfa8 _pthread_start + 148
16 libsystem_pthread.dylib 0x0000000199ce6da0 thread_start + 8
I would appreciate your help in understanding how to work with these type of dropdowns to filter the dynamic table. Thanks in advance.