fsspec - way to use proxy to connect to sftp in Python

Question

authentication_kwargs = dict()
authentication_kwargs["password"] = password
sftp = fsspec.filesystem("sftp", host=host, port=port, username=username,**authentication_kwargs)

This is how I connect to sftp using host, port, username and password.

How can I use proxy host and proxy port here?

example: proxy host: proxy.abo.quinn.co proxy port: 8081

Answer 1

You can do this by sub-classing the SFTPFileSystem class, overloading it's _connect method and using a paramiko.Transport to create the connection (which supports proxies):

from fsspec.implementations.sftp import SFTPFileSystem
from fsspec.registry import register_implementation

import paramiko

PROXY_CMD = '/usr/lib/ssh/ssh-http-proxy-connect'

class SFTPProxyFileSystem(SFTPFileSystem):

    protocol = "sftp_proxy"

    def _connect(self):
        port = self.ssh_kwargs.get('port', 22)
        username = self.ssh_kwargs.get('username')
        password = self.ssh_kwargs.get('password')

        proxy_host = self.ssh_kwargs.get('proxy_host')
        proxy_port = self.ssh_kwargs.get('proxy_port')

        proxy_cmd = '{} -h {} -p {} {} {}'.format(PROXY_CMD, proxy_host, proxy_port, self.host, port)
        proxy = paramiko.ProxyCommand(proxy_cmd)

        transport = paramiko.Transport(proxy, (self.host, port))
        transport.connect(username=username, password=password)

        self.client = paramiko.SFTPClient.from_transport(transport)
        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.ftp = self.client.open_sftp()

You should then be able to use below code snippet to connect to an sftp server using a proxy:

    register_implementation('sftp_proxy', SFTPProxyFileSystem)

    authentication_kwargs = dict()
    authentication_kwargs["password"] = "password"
    sftp = fsspec.filesystem("sftp_proxy", host="host", port=22, username="username",\
        proxy_host="proxy_host", proxy_port=3128, **authentication_kwargs)