Ignoring specific hosts in mitmproxy programatically

Question

I´m trying to create a proxy to intercept very specific requests, i got it to work without incidents, however when the website is loading there is a problem with 2 specific hosts which i´m trying to ignore by using "ignore_hosts" option what i have tried so far is:

import asyncio
from mitmproxy import options
from mitmproxy.tools import dump


class RequestLogger:
    def request(self, flow):
        print(flow.request.host)



ig = ['localhost', '127.0.0.0/8', '10.0.0.0/8', '192.168.0.0/16', '172.16.0.0/12' , '*.localdomain.com' ]

async def start_proxy(host,port):
    opts = options.Options()
    opts.update(listen_host=host, listen_port=port, ignore_hosts=ig)
    master = dump.DumpMaster(opts,with_termlog=True,with_dumper=True)
    master.addons.add(RequestLogger())
    
    await master.run()
    return master

if __name__ == '__main__':
    asyncio.run(start_proxy("0.0.0.0",8080))

But I keep getting:

   [21:12:58.880] Addon error: nothing to repeat at position 0
    Traceback (most recent call last):
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 111, in next_layer
        nextlayer.layer = self._next_layer(
                          ^^^^^^^^^^^^^^^^^
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 133, in _next_layer
        if self._ignore_connection(context, data_client, data_server):
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 244, in _ignore_connection
        return any(
               ^^^^
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 245, in <genexpr>
        re.search(rex, host, re.IGNORECASE)
      File "C:\Python\Lib\re\__init__.py", line 176, in search
        return _compile(pattern, flags).search(string)
               ^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\__init__.py", line 294, in _compile
        p = _compiler.compile(pattern, flags)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_compiler.py", line 745, in compile
        p = _parser.parse(p, flags)
            ^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_parser.py", line 989, in parse
        p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_parser.py", line 464, in _parse_sub
        itemsappend(_parse(source, state, verbose, nested + 1,
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_parser.py", line 691, in _parse
        raise source.error("nothing to repeat",
    re.error: nothing to repeat at position 0
    [21:12:59.006] Addon error: nothing to repeat at position 0
    Traceback (most recent call last):
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 111, in next_layer
        nextlayer.layer = self._next_layer(
                          ^^^^^^^^^^^^^^^^^
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 133, in _next_layer
        if self._ignore_connection(context, data_client, data_server):
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 244, in _ignore_connection
        return any(
               ^^^^
      File "c:\Users\PhenomAmd\AppData\Local\pypoetry\Cache\virtualenvs\proxylsf-Ra9sO3Vd-py3.11\Lib\site-packages\mitmproxy\addons\next_layer.py", line 245, in <genexpr>
        re.search(rex, host, re.IGNORECASE)
      File "C:\Python\Lib\re\__init__.py", line 176, in search
        return _compile(pattern, flags).search(string)
               ^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\__init__.py", line 294, in _compile
        p = _compiler.compile(pattern, flags)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_compiler.py", line 745, in compile
        p = _parser.parse(p, flags)
            ^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_parser.py", line 989, in parse
        p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_parser.py", line 464, in _parse_sub
        itemsappend(_parse(source, state, verbose, nested + 1,
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "C:\Python\Lib\re\_parser.py", line 691, in _parse
        raise source.error("nothing to repeat",
    re.error: nothing to repeat at position 0

I have also tried to do:

  

ignore=[]
hosts = ['host1','host2','host3']
for host in hosts:
    ignore.append(host)


opts.update(listen_host=host, listen_port=port, ignore_hosts=ignore)

any help would be appreciated

Answer 1

Most likely this is caused by the content of your ignore hosts list:

['localhost', '127.0.0.0/8', '10.0.0.0/8', '192.168.0.0/16', '172.16.0.0/12' , '*.localdomain.com' ]

Mitmproxy expects regular expressions, not network definitions like 127.0.0.0/8 or strings with * as wildcard.

You have to change your ignore hosts to regular expressions, for example

• 127.0.0.0/8 -> 127\.[0-9]+\.[0-9]+\.[0-9]+\
• *.localdomain.com-> .*\.localdomain\.com.