answer to my question..after multiples tentative for solving it. One finally works.
The code works on google colab and kaggle notebooks (kaggle needs login and turning internet on (disabled by default)
adjusting answer:
added steps
- Uninstall chromium-browser if it was installed by default or previously
- Install google-chrome-stable
- import os, requests, zipfile, io
- Get the installed Chrome browser version
- Dynamically get ChromeDriver download URL from Chrome for Testing API
- Fetch the list of known good versions from the Chrome for Testing API
- Find the latest stable ChromeDriver for the detected major version
- Enable, Move and extract dynamically the extracted_folder
code below
# =========================================
# STEP 1 — INSTALL DEPENDENCIES & DRIVER CONFIGURATION
# =========================================
!apt-get update -qq
# Uninstall chromium-browser if it was installed by default or previously
!apt-get remove -y chromium-browser > /dev/null 2>&1
# Install google-chrome-stable
!wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
!echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
!apt-get update -qq
!apt-get install -qq google-chrome-stable
!pip install -q selenium tabulate ipywidgets pillow pandas requests openpyxl
import os
import requests
import zipfile
import io
import re # Import re module
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
# Get the installed Chrome browser version
try:
chrome_version_output = os.popen('google-chrome --version').read()
chrome_version_match = re.search(r'Google Chrome (\d+)', chrome_version_output)
if not chrome_version_match:
# Fallback to chromium-browser if google-chrome version not found
chrome_version_output = os.popen('chromium-browser --version').read()
chrome_version_match = re.search(r'Chromium (\d+)', chrome_version_output)
if chrome_version_match:
chrome_major_version = chrome_version_match.group(1)
print(f"Detected Chrome major version: {chrome_major_version}")
else:
raise ValueError("Could not determine Chrome major version.")
except Exception as e:
print(f"Error detecting Chrome version: {e}")
# Default to a known good version if detection fails in Colab
chrome_major_version = "123" # A common version in Colab, adjust if needed
print(f"Defaulting to Chrome major version: {chrome_major_version}")
# Dynamically get ChromeDriver download URL from Chrome for Testing API
chromedriver_url = None
try:
# Fetch the list of known good versions from the Chrome for Testing API
response = requests.get("https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json")
response.raise_for_status()
versions_data = response.json()
# Find the latest stable ChromeDriver for the detected major version
for version_info in reversed(versions_data['versions']):
if version_info['version'].startswith(f"{chrome_major_version}."):
for download in version_info['downloads']['chromedriver']:
if download['platform'] == 'linux64':
chromedriver_url = download['url']
break
if chromedriver_url:
break
if not chromedriver_url:
raise ValueError(f"Could not find ChromeDriver download URL for Chrome major version {chrome_major_version}")
print(f"Downloading ChromeDriver from: {chromedriver_url}")
response = requests.get(chromedriver_url)
response.raise_for_status() # Raise an exception for bad status codes
with zipfile.ZipFile(io.BytesIO(response.content)) as z:
z.extractall("/tmp/") # Extract to a temporary directory
# Move chromedriver to /usr/bin and make it executable
# The extracted folder name might vary, so find it dynamically
extracted_folder = [f for f in os.listdir('/tmp') if 'chromedriver' in f and os.path.isdir(os.path.join('/tmp', f))][0]
!mv /tmp/{extracted_folder}/chromedriver /usr/bin/chromedriver
!chmod +x /usr/bin/chromedriver
print("✅ ChromeDriver downloaded and configured.")
except Exception as e:
print(f"❌ Error downloading or configuring ChromeDriver: {e}")
print("Please check the Chrome version and the corresponding ChromeDriver download URL.")
raise # Re-raise to halt execution if setup fails
# =========================================
# STEP 2 — IMPORTS
# =========================================
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import StaleElementReferenceException, TimeoutException
import ipywidgets as widgets
from IPython.display import display, clear_output, HTML
import pandas as pd
import requests, time, os, re
from datetime import datetime
# =========================================
# STEP 3 — DRIVER CONFIGURATION
# =========================================
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--disable-gpu") # Added for stability in headless mode
chrome_options.add_argument("--window-size=1920,1080")
# Configure ChromeDriver service with the now-installed chromedriver
service = Service(executable_path="/usr/bin/chromedriver")
driver = webdriver.Chrome(service=service, options=chrome_options)
wait = WebDriverWait(driver, 15)
print("✅ Selenium WebDriver initialized.")
