python asyncio가 3.4에서 3.5+로 마이 그 레이션

Question

여러분 안녕하세요. 저는 인터넷 로봇을 만들려고 노력하고 있습니다. 스크립트를 Python 3.4에서 3.5 또는 3.6+로 마이그레이션하는 동안 문제를 만났습니다. 그것은 asyncio 사용하고, 3.4 파이썬에 잘 작동하지만 내가 python3.5로 시작 + 내가 오류를 받았을 때 : 여기

RuntimeError: Cannot run the event loop while another loop is running 코드 체계입니다 :

import multiprocessing as mp 
import asyncio 
import concurrent.futures 
import aiohttp 

def create_proccesses(separate_loop_creator, coro): 
    proccesses = [] 
    for n in range(2): 
     proc = mp.Process(target=separate_loop_creator, args=(coro,)) 
     proc.start() 
     proccesses.append(proc) 
    for p in proccesses: 
     p.join() 

def separate_loop_creator(coro): 
    sep_loop = asyncio.new_event_loop() 
    asyncio.set_event_loop(sep_loop) 
    tasks = [asyncio.async(coro(sep_loop)) for _ in range(100)] 
    try: 
     sep_loop.run_until_complete(asyncio.wait(tasks)) 
     sep_loop.close() 
    except Exception as err: 
     print(err) 
     for task in tasks: 
      task.cancel() 
     sep_loop.close() 


@asyncio.coroutine 
def manager(exe, loop): 
    # some calculations and start coros in several processes 
    loop.run_in_executor(
     exe, 
     create_proccesses, 
     separate_loop_creator, 
     some_coro 
    ) 

@asyncio.coroutine 
def some_work_in_mainloop(): 
    while True: 
     print('Some server dealing with connections here...') 
     yield from asyncio.sleep(1) 

@asyncio.coroutine 
def some_coro(loop): 
    with aiohttp.ClientSession(loop=loop) as session: 
     response = yield from session.get('http://google.com') 
     yield from asyncio.sleep(2) 
     print(response.status) 

if __name__ == '__main__': 
    mainloop = asyncio.get_event_loop() 
    executor = concurrent.futures.ProcessPoolExecutor(5) 
    asyncio.async(some_work_in_mainloop()) 
    asyncio.async(manager(executor, mainloop)) 
    try: 
     mainloop.run_forever() 
    finally: 
     mainloop.close() 

예외가 separate_loop_creator() 코 루틴에 제기하고있다 RuntimeError: Cannot run the event loop while another loop is running. 나는 get_event_loop() mechnics를 변경했기 때문에 그런 것 같지만 코드에 무엇이 잘못된 것인지 이해하지 못합니다. 여기

는 내가하고 싶은 것입니다 :

Traceback (most recent call last): 
    File "tst.py", line 21, in separate_loop_creator 
    sep_loop.run_until_complete(asyncio.wait(tasks)) 
    File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 454, in run_until_complete 
    self.run_forever() 
    File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 411, in run_forever 
    'Cannot run the event loop while another loop is running') 
RuntimeError: Cannot run the event loop while another loop is running 
Cannot run the event loop while another loop is running 
Traceback (most recent call last): 
    File "tst.py", line 21, in separate_loop_creator 
    sep_loop.run_until_complete(asyncio.wait(tasks)) 
    File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 454, in run_until_complete 
    self.run_forever() 
    File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 411, in run_forever 
    'Cannot run the event loop while another loop is running') 
RuntimeError: Cannot run the event loop while another loop is running 

파이썬 3.4.3 결과 :

... 
200 
Some server dealing with connections here... 
200 
200 
Some server dealing with connections here... 
200 
200 
Some server dealing with connections here... 
200 
... 

Answer 1

사실 이것은 CPython 3.6.0에서 asyncio의 버그입니다. 이 문제를 해결하기 위해 PR이있어 3.6.1이 예상대로 작동합니다. 프로젝트에서 다음 코드 조각을 추가 할 수있는 해결 방법으로

가 :

import sys 

if sys.version_info[:3] == (3, 6, 0): 
    import asyncio.events as _ae 
    import os as _os 

    _ae._RunningLoop._pid = None 

    def _get_running_loop(): 
     if _ae._running_loop._pid == _os.getpid(): 
      return _ae._running_loop._loop 

    def _set_running_loop(loop): 
     _ae._running_loop._pid = _os.getpid() 
     _ae._running_loop._loop = loop 

    _ae._get_running_loop = _get_running_loop 
    _ae._set_running_loop = _set_running_loop 

Answer 2

다음

     +--------------+ 
       +-------+other service | 
    +----------+  +--------------+ 
    | mainloop | 
    +----------+ 
       |  +------------+ 
       +-----+ executor | 
        +------+-----+ 
          | 
        +------+--------+ 
        |start proccess | 
        +---+-------+---+ 
+-----------------+  |  |  +---------------+ 
|start new loop +------+  +------+ start new loop| 
+--------+--------+      +-------+-------+ 
     |          | 
+-------+-------+      +------v-------+ 
| run coro |      | run coro  | 
+---------------+      +--------------+ 

내가 python3.5.3에 도착 추적이다 가능한 한 최선의 해결책은 프로그램에서 multiprocessing을 모두 제거하고 하나의 이벤트 루프 만 사용하십시오 (Optiona 격리 된 CPU 집약적 인 작업에 ProcessPoolExecutor을 사용합니다.

2017-03-02 현재이 문제에 대한 공개 python 버그가 있으며 비 Windows 플랫폼의 경우 https://bugs.python.org/issue22087입니다. 수정에서 영감을

import asyncio 
import multiprocessing as mp 


def create_another_loop(): 
    loop = asyncio.new_event_loop() 
    loop.run_forever() 


async def create_process(): 
    proc = mp.Process(target=create_another_loop) 
    proc.start() 
    proc.join() 


if __name__ == '__main__': 
    main_loop = asyncio.get_event_loop() 
    main_loop.run_until_complete(create_process()) 
    main_loop.close() 

hackish 해결 https://github.com/python/asyncio/pull/497 새로이 코드를 추가하는 것입니다 여기에 제안 (당신의 자신의 위험에 사용주의!!) : 여기

같은 문제를 유발하는 짧은 프로그램입니다 Process 생성 :

if asyncio.events._running_loop: 
    asyncio.events._running_loop._loop = None 

예 :

import asyncio 
import multiprocessing as mp 
import time 
from concurrent.futures.process import ProcessPoolExecutor 


async def clock(label, n=5, sleep=1): 
    print(label, "start") 
    for i in range(n): 
     await asyncio.sleep(sleep) 
     print(label, i + 1) 
    print(label, "end") 
    return label 


def create_another_loop(): 
    # HACK START 
    if asyncio.events._running_loop: 
     asyncio.events._running_loop._loop = None 
    # HACK END 

    loop = asyncio.new_event_loop() 
    loop.run_until_complete(clock("sub")) 
    loop.close() 


def create_process(): 
    time.sleep(2) 
    proc = mp.Process(target=create_another_loop) 
    proc.start() 
    proc.join() 
    return "ok" 


async def create_process_in_pool(): 
    return await main_loop.run_in_executor(ProcessPoolExecutor(), create_process) 


if __name__ == '__main__': 
    main_loop = asyncio.get_event_loop() 
    tasks = (
     clock("main"), 
     create_process_in_pool(), 
    ) 
    print(main_loop.run_until_complete(asyncio.gather(*tasks))) 
    main_loop.close() 

다른 가능한 해결 방법 : 루프를 시작하기 전에 프로세스를 생성하거나 communicate with the subprocess via a stream까지 허용하는 asyncio.create_subprocess_exec을 사용하십시오.