Selenium Chrome could not reach the internet from remote server

  docker, selenium-chromedriver

I am using Chrome driver Docker image within a docker-compose.yml.

The Python code running using Chrome is as follow:

class merge :

    def __init__(self,file_dir=None,personalized = 1,wait = 10) :
        self.wait = wait
        self.personalized = personalized
        if file_dir is None :
            file_dir = os.getcwd()
            print(file_dir)
            self.path = Path(file_dir)
        options = Options()
        options.add_argument("--no-sandbox")
        options.add_argument("--headless")
        options.add_argument("--window-size=1920,1080")
        options.add_argument("--disable-dev-shm-usage")
        options.add_argument("--allow-insecure-localhost")
        options.add_argument("--ignore-certificate-errors")
        prefs = {
            "download": {
                "default_directory": "/tmp",
                "prompt_for_download": False,
                "directory_upgrade": True,
            },
        }
        options.add_experimental_option("prefs",prefs)  
        self.drv = webdriver.Remote(command_executor='http://chrome:4444/wd/hub', options=options)

Docker-compose is running on a Linux server and output the error below. It used to run on my Windows 10 locally without any issue.

Traceback (most recent call last):
  File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line 1157, in _run_raw_task
    self._prepare_and_execute_task_with_callbacks(context, task)
  File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line 1331, in _prepare_and_execute_task_with_callbacks
    result = self._execute_task(context, task_copy)
  File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line 1361, in _execute_task
    result = task_copy.execute(context=context)
  File "/home/airflow/.local/lib/python3.6/site-packages/airflow/providers/papermill/operators/papermill.py", line 76, in execute
    report_mode=True,
  File "/home/airflow/.local/lib/python3.6/site-packages/papermill/execute.py", line 122, in execute_notebook
    raise_for_execution_errors(nb, output_path)
  File "/home/airflow/.local/lib/python3.6/site-packages/papermill/execute.py", line 234, in raise_for_execution_errors
    raise error
papermill.exceptions.PapermillExecutionError: 
---------------------------------------------------------------------------
Exception encountered at "In [2]":
---------------------------------------------------------------------------
WebDriverException                        Traceback (most recent call last)
<ipython-input-2-6c322585e07d> in <module>
----> 1 mmo = *************()
      2 mmo.open(tgt="http://www.**************.com/**************/exit.asp?")
      3 mmo.login('***********@*******.com','*************')

/opt/***/script/***_Tools/20210511_DATADOWN_RETRY/mergermarket_lib.py in __init__(self, file_dir, personalized, wait)
     55         options.add_experimental_option("prefs",prefs)
     56         #options.add_experimental_option("prefs",{"download.default_directory":file_dir})
---> 57         self.drv = webdriver.Remote(command_executor='http://chrome:4444/wd/hub', options=options)
     58 
     59 

~/.local/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py in __init__(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive, file_detector, options)
    155             warnings.warn("Please use FirefoxOptions to set browser profile",
    156                           DeprecationWarning, stacklevel=2)
--> 157         self.start_session(capabilities, browser_profile)
    158         self._switch_to = SwitchTo(self)
    159         self._mobile = Mobile(self)

~/.local/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py in start_session(self, capabilities, browser_profile)
    250         parameters = {"capabilities": w3c_caps,
    251                       "desiredCapabilities": capabilities}
--> 252         response = self.execute(Command.NEW_SESSION, parameters)
    253         if 'sessionId' not in response:
    254             response = response['value']

~/.local/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
    319         response = self.command_executor.execute(driver_command, params)
    320         if response:
--> 321             self.error_handler.check_response(response)
    322             response['value'] = self._unwrap_value(
    323                 response.get('value', None))

~/.local/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
    240                 alert_text = value['alert'].get('text')
    241             raise exception_class(message, screen, stacktrace, alert_text)
--> 242         raise exception_class(message, screen, stacktrace)
    243 
    244     def _value_or_default(self, obj, key, default):

WebDriverException: Message: unknown error: cannot create temp dir for user data dir
Build info: version: '3.141.59', revision: 'e82be7d358', time: '2018-11-14T08:25:53'
System info: host: 'chrome', ip: '172.19.0.4', os.name: 'Linux', os.arch: 'amd64', os.version: '3.10.0-1160.36.2.el7.x86_64', java.version: '1.8.0_292'
Driver info: driver.version: unknown
remote stacktrace: #0 0x555e14e86583 <unknown>
#1 0x555e14bfaa38 <unknown>
#2 0x555e14c1b90d <unknown>
#3 0x555e14c194df <unknown>
#4 0x555e14c531e2 <unknown>
#5 0x555e14c4d643 <unknown>
#6 0x555e14c23b94 <unknown>
#7 0x555e14c24b85 <unknown>
#8 0x555e14eb6c6e <unknown>
#9 0x555e14ecc975 <unknown>
#10 0x555e14eb7bc5 <unknown>
#11 0x555e14ecdfb5 <unknown>
#12 0x555e14eac84b <unknown>
#13 0x555e14ee9248 <unknown>
#14 0x555e14ee93c8 <unknown>
#15 0x555e14f0373d <unknown>
#16 0x2b8d10211609 start_thread

Stacktrace:
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0 (NativeConstructorAccessorImpl.java:-2)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance (NativeConstructorAccessorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance (DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance (Constructor.java:423)
    at org.openqa.selenium.remote.W3CHandshakeResponse.lambda$errorHandler$0 (W3CHandshakeResponse.java:62)
    at org.openqa.selenium.remote.HandshakeResponse.lambda$getResponseFunction$0 (HandshakeResponse.java:30)
    at org.openqa.selenium.remote.ProtocolHandshake.lambda$createSession$0 (ProtocolHandshake.java:126)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.Spliterators$ArraySpliterator.tryAdvance (Spliterators.java:958)
    at java.util.stream.ReferencePipeline.forEachWithCancel (ReferencePipeline.java:126)
    at java.util.stream.AbstractPipeline.copyIntoWithCancel (AbstractPipeline.java:499)
    at java.util.stream.AbstractPipeline.copyInto (AbstractPipeline.java:486)
    at java.util.stream.AbstractPipeline.wrapAndCopyInto (AbstractPipeline.java:472)
    at java.util.stream.FindOps$FindOp.evaluateSequential (FindOps.java:152)
    at java.util.stream.AbstractPipeline.evaluate (AbstractPipeline.java:234)
    at java.util.stream.ReferencePipeline.findFirst (ReferencePipeline.java:531)
    at org.openqa.selenium.remote.ProtocolHandshake.createSession (ProtocolHandshake.java:128)
    at org.openqa.selenium.remote.ProtocolHandshake.createSession (ProtocolHandshake.java:74)
    at org.openqa.selenium.grid.session.remote.RemoteSession$Factory.performHandshake (RemoteSession.java:147)
    at org.openqa.selenium.grid.session.remote.ServicedSession$Factory.apply (ServicedSession.java:161)
    at org.openqa.selenium.remote.server.ActiveSessionFactory.lambda$apply$12 (ActiveSessionFactory.java:180)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.stream.ReferencePipeline$11$1.accept (ReferencePipeline.java:440)
    at java.util.stream.ReferencePipeline$2$1.accept (ReferencePipeline.java:175)
    at java.util.Spliterators$ArraySpliterator.tryAdvance (Spliterators.java:958)
    at java.util.stream.ReferencePipeline.forEachWithCancel (ReferencePipeline.java:126)
    at java.util.stream.AbstractPipeline.copyIntoWithCancel (AbstractPipeline.java:499)
    at java.util.stream.AbstractPipeline.copyInto (AbstractPipeline.java:486)
    at java.util.stream.AbstractPipeline.wrapAndCopyInto (AbstractPipeline.java:472)
    at java.util.stream.FindOps$FindOp.evaluateSequential (FindOps.java:152)
    at java.util.stream.AbstractPipeline.evaluate (AbstractPipeline.java:234)
    at java.util.stream.ReferencePipeline.findFirst (ReferencePipeline.java:531)
    at org.openqa.selenium.remote.server.ActiveSessionFactory.apply (ActiveSessionFactory.java:183)
    at org.openqa.selenium.remote.server.NewSessionPipeline.lambda$null$2 (NewSessionPipeline.java:66)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.stream.ReferencePipeline$2$1.accept (ReferencePipeline.java:175)
    at java.util.Collections$2.tryAdvance (Collections.java:4719)
    at java.util.stream.ReferencePipeline.forEachWithCancel (ReferencePipeline.java:126)
    at java.util.stream.AbstractPipeline.copyIntoWithCancel (AbstractPipeline.java:499)
    at java.util.stream.AbstractPipeline.copyInto (AbstractPipeline.java:486)
    at java.util.stream.AbstractPipeline.wrapAndCopyInto (AbstractPipeline.java:472)
    at java.util.stream.FindOps$FindOp.evaluateSequential (FindOps.java:152)
    at java.util.stream.AbstractPipeline.evaluate (AbstractPipeline.java:234)
    at java.util.stream.ReferencePipeline.findFirst (ReferencePipeline.java:531)
    at org.openqa.selenium.remote.server.NewSessionPipeline.lambda$createNewSession$3 (NewSessionPipeline.java:69)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.stream.DistinctOps$1$2.accept (DistinctOps.java:175)
    at java.util.stream.ReferencePipeline$2$1.accept (ReferencePipeline.java:175)
    at java.util.stream.ReferencePipeline$3$1.accept (ReferencePipeline.java:193)
    at java.util.stream.ReferencePipeline$2$1.accept (ReferencePipeline.java:175)
    at java.util.stream.Streams$StreamBuilderImpl.tryAdvance (Streams.java:405)
    at java.util.stream.Streams$ConcatSpliterator.tryAdvance (Streams.java:728)
    at java.util.stream.ReferencePipeline.forEachWithCancel (ReferencePipeline.java:126)
    at java.util.stream.AbstractPipeline.copyIntoWithCancel (AbstractPipeline.java:499)
    at java.util.stream.AbstractPipeline.copyInto (AbstractPipeline.java:486)
    at java.util.stream.AbstractPipeline.wrapAndCopyInto (AbstractPipeline.java:472)
    at java.util.stream.FindOps$FindOp.evaluateSequential (FindOps.java:152)
    at java.util.stream.AbstractPipeline.evaluate (AbstractPipeline.java:234)
    at java.util.stream.ReferencePipeline.findFirst (ReferencePipeline.java:531)
    at org.openqa.selenium.remote.server.NewSessionPipeline.createNewSession (NewSessionPipeline.java:72)
    at org.openqa.selenium.remote.server.commandhandler.BeginSession.execute (BeginSession.java:65)
    at org.openqa.selenium.remote.server.WebDriverServlet.lambda$handle$0 (WebDriverServlet.java:235)
    at java.util.concurrent.Executors$RunnableAdapter.call (Executors.java:511)
    at java.util.concurrent.FutureTask.run (FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:624)
    at java.lang.Thread.run (Thread.java:748)

[2021-09-15 02:26:57,532] {taskinstance.py:1551} INFO - Marking task as FAILED. dag_id=PTS_WORFLOW, task_id=20210511_MMLIB, execution_date=20210915T022652, start_date=20210915T022654, end_date=20210915T022657
[2021-09-15 02:26:57,562] {local_task_job.py:149} INFO - 

Task exited with return code 1

Is there something obvious I am missing which prevent the connectivity?

Source: Docker Questions

LEAVE A COMMENT