Issue 35919: multiprocessing: shared manager Pool fails with AttributeError (original) (raw)

import multiprocessing import multiprocessing.managers

def f(n): return n * n

def worker(pool): with pool: pool.apply_async(f, (10, ))

manager = multiprocessing.managers.SyncManager() manager.start() pool = manager.Pool(processes=4) proc = multiprocessing.Process(target=worker, args=(pool, )) proc.start() proc.join()

This is related to BPO-35917 and it fails with:

Process Process-2: Traceback (most recent call last): File "/home/giampaolo/cpython/Lib/multiprocessing/process.py", line 302, in _bootstrap self.run() File "/home/giampaolo/cpython/Lib/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "foo.py", line 54, in worker pool.apply_async(f, (10, )) File "", line 2, in apply_async File "/home/giampaolo/cpython/Lib/multiprocessing/managers.py", line 802, in _callmethod proxytype = self._manager._registry[token.typeid][-1] AttributeError: 'NoneType' object has no attribute '_registry'

import traceback import multiprocessing.managers

class MyManager(multiprocessing.managers.SyncManager): pass

class DictList(multiprocessing.managers.BaseProxy): method_to_typeid = {'getitem': 'dict'}

def __getitem__(self, key):
    return self._callmethod('__getitem__', (key,))

MyManager.register('DictList', None, DictList)

with MyManager() as manager:

nested = manager.DictList([{'hello': 'world'}])
print(nested[0]['hello'])  # world

proxy = manager.list([nested])
try:
    print(proxy[0][0]['hello'])
except AttributeError:
    traceback.print_exc()
    print("""
    Bug: AttributeError: ProxyBase._callmethod is None
    --------------------------------------------------

    This error is raised because proxies returned as #RETURN messages
    have no manager reference, which is required to resolve typeids
    (using BaseManager._registry).

    Only proxies returned as #PROXY messages will be fully functional.

    This is an undocumented current implementation limitation.

    Fix (proposal)
    --------------

    Include the manager class (not the instance) as part of the proxy
    serialization in BaseProxy.__reduce__, as BaseManager._registry is
    a class variable.

    Note: #PROXY message protocol can be also replaced by regular proxy
          serialization after this fix, resulting on simpler codebase.
    """)