From 77fa12dfcb0aa990c15886de794a43fe8967d90b Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Fri, 21 Oct 2016 14:04:45 +0200 Subject: [PATCH 1/6] Remove all occurrences of "loop=loop" --- examples/aiohttp_client.py | 2 +- examples/http_client.py | 2 +- examples/producer_consumer.py | 2 +- examples/tcp_echo_client.py | 3 +-- examples/tcp_echo_server.py | 2 +- webscraper.rst | 3 +-- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/aiohttp_client.py b/examples/aiohttp_client.py index 7602c8b..4ca4da2 100644 --- a/examples/aiohttp_client.py +++ b/examples/aiohttp_client.py @@ -25,7 +25,7 @@ def get_multiple_pages(host, waits, port=8000, show_time=True): pages = [] start = time.perf_counter() with closing(asyncio.get_event_loop()) as loop: - with aiohttp.ClientSession(loop=loop) as session: + with aiohttp.ClientSession() as session: for wait in waits: tasks.append(fetch_page(session, host, port, wait)) pages = loop.run_until_complete(asyncio.gather(*tasks)) diff --git a/examples/http_client.py b/examples/http_client.py index 72987bf..430f283 100644 --- a/examples/http_client.py +++ b/examples/http_client.py @@ -8,7 +8,7 @@ async def fetch_page(session, url): return await response.read() loop = asyncio.get_event_loop() -with aiohttp.ClientSession(loop=loop) as session: +with aiohttp.ClientSession() as session: content = loop.run_until_complete( fetch_page(session, 'http://python.org')) print(content) diff --git a/examples/producer_consumer.py b/examples/producer_consumer.py index 459b021..95ac384 100644 --- a/examples/producer_consumer.py +++ b/examples/producer_consumer.py @@ -31,7 +31,7 @@ async def consume(queue): loop = asyncio.get_event_loop() -queue = asyncio.Queue(loop=loop) +queue = asyncio.Queue() producer_coro = produce(queue, 10) consumer_coro = consume(queue) loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) diff --git a/examples/tcp_echo_client.py b/examples/tcp_echo_client.py index e941029..71534c3 100644 --- a/examples/tcp_echo_client.py +++ b/examples/tcp_echo_client.py @@ -2,8 +2,7 @@ async def tcp_echo_client(message, loop): - reader, writer = await asyncio.open_connection('127.0.0.1', 8888, - loop=loop) + reader, writer = await asyncio.open_connection('127.0.0.1', 8888) print('Send: %r' % message) writer.write(message.encode()) diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index d6ac5c7..f7f58d0 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -14,7 +14,7 @@ async def handle_echo(reader, writer): writer.close() loop = asyncio.get_event_loop() -coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop) +coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888) server = loop.run_until_complete(coro) # Serve requests until Ctrl+C is pressed diff --git a/webscraper.rst b/webscraper.rst index f85fa7a..df50bcd 100644 --- a/webscraper.rst +++ b/webscraper.rst @@ -397,7 +397,7 @@ This is the interesting part of ``get_multiple_pages()``: .. code-block:: python with closing(asyncio.get_event_loop()) as loop: - with aiohttp.ClientSession(loop=loop) as session: + with aiohttp.ClientSession() as session: for wait in waits: tasks.append(fetch_page(session, host, port, wait)) pages = loop.run_until_complete(asyncio.gather(*tasks)) @@ -427,4 +427,3 @@ It also takes about five seconds and gives the same output as our version before. But the implementation for getting a single page is much simpler and takes care of the encoding and other aspects not mentioned here. - From f6ba98d7c2711a9db33328bd0597689214e7046f Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Fri, 18 Nov 2016 16:36:34 +0100 Subject: [PATCH 2/6] Update examples to promote the use of a main coroutine --- examples/asyncio_deferred.py | 4 +--- examples/create_task.py | 9 ++++++--- examples/hello_clock.py | 12 +++++++----- examples/hello_world.py | 2 ++ examples/http_client.py | 13 +++++++++---- examples/loop_stop.py | 18 +++++++++++------- examples/producer_consumer.py | 29 +++++++++++++++++------------ examples/producer_consumer_join.py | 8 +++----- examples/run_in_thread.py | 2 +- examples/subprocess_command.py | 22 +++++++++++++++------- examples/subprocess_echo.py | 3 +++ examples/tcp_echo_client.py | 10 +++++++--- examples/tcp_echo_server.py | 21 ++++++++++++++++----- 13 files changed, 98 insertions(+), 55 deletions(-) diff --git a/examples/asyncio_deferred.py b/examples/asyncio_deferred.py index 21b2f1f..900b6aa 100644 --- a/examples/asyncio_deferred.py +++ b/examples/asyncio_deferred.py @@ -13,7 +13,5 @@ async def steps(x): loop = asyncio.get_event_loop() -coro = steps(5) -loop.run_until_complete(coro) +loop.run_until_complete(steps(5)) loop.close() - diff --git a/examples/create_task.py b/examples/create_task.py index 3f096ff..9543406 100644 --- a/examples/create_task.py +++ b/examples/create_task.py @@ -1,14 +1,17 @@ import asyncio + async def say(what, when): await asyncio.sleep(when) print(what) -loop = asyncio.get_event_loop() +async def schedule(): + asyncio.ensure_future(say('first hello', 2)) + asyncio.ensure_future(say('second hello', 1)) -loop.create_task(say('first hello', 2)) -loop.create_task(say('second hello', 1)) +loop = asyncio.get_event_loop() +loop.run_until_complete(schedule()) loop.run_forever() loop.close() diff --git a/examples/hello_clock.py b/examples/hello_clock.py index ca178e2..e9342ec 100644 --- a/examples/hello_clock.py +++ b/examples/hello_clock.py @@ -2,7 +2,6 @@ async def print_every_second(): - "Print seconds" while True: for i in range(60): print(i, 's') @@ -15,9 +14,12 @@ async def print_every_minute(): print(i, 'minute') +async def main(): + coro_second = print_every_second() + coro_minute = print_every_minute() + await asyncio.gather(coro_second, coro_minute) + + loop = asyncio.get_event_loop() -loop.run_until_complete( - asyncio.gather(print_every_second(), - print_every_minute()) -) +loop.run_until_complete(main()) loop.close() diff --git a/examples/hello_world.py b/examples/hello_world.py index 15d53bf..036f03d 100644 --- a/examples/hello_world.py +++ b/examples/hello_world.py @@ -1,9 +1,11 @@ import asyncio + async def say(what, when): await asyncio.sleep(when) print(what) + loop = asyncio.get_event_loop() loop.run_until_complete(say('hello world', 1)) loop.close() diff --git a/examples/http_client.py b/examples/http_client.py index 430f283..7362212 100644 --- a/examples/http_client.py +++ b/examples/http_client.py @@ -1,15 +1,20 @@ import asyncio import aiohttp + async def fetch_page(session, url): with aiohttp.Timeout(10): async with session.get(url) as response: assert response.status == 200 return await response.read() + +async def main(): + with aiohttp.ClientSession() as session: + content = await fetch_page(session, 'http://python.org') + print(content.decode()) + + loop = asyncio.get_event_loop() -with aiohttp.ClientSession() as session: - content = loop.run_until_complete( - fetch_page(session, 'http://python.org')) - print(content) +loop.run_until_complete(main()) loop.close() diff --git a/examples/loop_stop.py b/examples/loop_stop.py index fca7019..b113572 100644 --- a/examples/loop_stop.py +++ b/examples/loop_stop.py @@ -1,20 +1,24 @@ import asyncio + async def say(what, when): await asyncio.sleep(when) print(what) -async def stop_after(loop, when): + +async def stop_after(when): await asyncio.sleep(when) - loop.stop() + asyncio.get_event_loop().stop() -loop = asyncio.get_event_loop() +async def schedule(): + asyncio.ensure_future(say('first hello', 2)) + asyncio.ensure_future(say('second hello', 1)) + asyncio.ensure_future(say('third hello', 4)) + asyncio.ensure_future(stop_after(3)) -loop.create_task(say('first hello', 2)) -loop.create_task(say('second hello', 1)) -loop.create_task(say('third hello', 4)) -loop.create_task(stop_after(loop, 3)) +loop = asyncio.get_event_loop() +loop.run_until_complete(schedule()) loop.run_forever() loop.close() diff --git a/examples/producer_consumer.py b/examples/producer_consumer.py index 95ac384..a93994e 100644 --- a/examples/producer_consumer.py +++ b/examples/producer_consumer.py @@ -3,12 +3,13 @@ async def produce(queue, n): - for x in range(1, n + 1): - # produce an item + # produce n items + for x in range(1, n+1): + + # produce an item (simulate i/o operation using sleep) print('producing {}/{}'.format(x, n)) - # simulate i/o operation using sleep - await asyncio.sleep(random.random()) - item = str(x) + item = await asyncio.sleep(random.random(), result=x) + # put the item in the queue await queue.put(item) @@ -20,19 +21,23 @@ async def consume(queue): while True: # wait for an item from the producer item = await queue.get() + + # the producer emits None to indicate that it is done if item is None: - # the producer emits None to indicate that it is done break - # process the item + # process the item (simulate i/o operation using sleep) print('consuming item {}...'.format(item)) - # simulate i/o operation using sleep await asyncio.sleep(random.random()) +async def main(): + queue = asyncio.Queue() + producer_coro = produce(queue, 10) + consumer_coro = consume(queue) + await asyncio.gather(producer_coro, consumer_coro) + + loop = asyncio.get_event_loop() -queue = asyncio.Queue() -producer_coro = produce(queue, 10) -consumer_coro = consume(queue) -loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/producer_consumer_join.py b/examples/producer_consumer_join.py index 5af4db6..30a5eb3 100644 --- a/examples/producer_consumer_join.py +++ b/examples/producer_consumer_join.py @@ -17,22 +17,20 @@ async def consume(queue): while True: # wait for an item from the producer item = await queue.get() - # process the item print('consuming {}...'.format(item)) # simulate i/o operation using sleep await asyncio.sleep(random.random()) - # Notify the queue that the item has been processed queue.task_done() -async def run(n): +async def main(): queue = asyncio.Queue() # schedule the consumer consumer = asyncio.ensure_future(consume(queue)) # run the producer and wait for completion - await produce(queue, n) + await produce(queue, 10) # wait until the consumer has processed all items await queue.join() # the consumer is still awaiting for an item, cancel it @@ -40,5 +38,5 @@ async def run(n): loop = asyncio.get_event_loop() -loop.run_until_complete(run(10)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/run_in_thread.py b/examples/run_in_thread.py index 5a704dc..3d24eb6 100644 --- a/examples/run_in_thread.py +++ b/examples/run_in_thread.py @@ -2,7 +2,7 @@ def compute_pi(digits): - # implementation + # CPU-intensive computation return 3.14 diff --git a/examples/subprocess_command.py b/examples/subprocess_command.py index d39b53c..0d7bb0d 100644 --- a/examples/subprocess_command.py +++ b/examples/subprocess_command.py @@ -5,19 +5,27 @@ async def run_command(*args): # Create subprocess process = await asyncio.create_subprocess_exec( *args, - # stdout must a pipe to be accessible as process.stdout + # stdout must be piped to be accessible as process.stdout stdout=asyncio.subprocess.PIPE) + # Wait for the subprocess to finish stdout, stderr = await process.communicate() + # Return stdout return stdout.decode().strip() +async def main(): + # Gather uname and date commands + commands = asyncio.gather(run_command('uname'), run_command('date')) + + # Wait for the results + uname, date = await commands + + # Print a report + print('uname: {}, date: {}'.format(uname, date)) + + loop = asyncio.get_event_loop() -# Gather uname and date commands -commands = asyncio.gather(run_command('uname'), run_command('date')) -# Run the commands -uname, date = loop.run_until_complete(commands) -# Print a report -print('uname: {}, date: {}'.format(uname, date)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/subprocess_echo.py b/examples/subprocess_echo.py index 7640d8c..fdeceba 100644 --- a/examples/subprocess_echo.py +++ b/examples/subprocess_echo.py @@ -9,13 +9,16 @@ async def echo(msg): stdin=asyncio.subprocess.PIPE, # stdout must a pipe to be accessible as process.stdout stdout=asyncio.subprocess.PIPE) + # Write message print('Writing {!r} ...'.format(msg)) process.stdin.write(msg.encode() + b'\n') + # Read reply data = await process.stdout.readline() reply = data.decode().strip() print('Received {!r}'.format(reply)) + # Stop the subprocess process.terminate() code = await process.wait() diff --git a/examples/tcp_echo_client.py b/examples/tcp_echo_client.py index 71534c3..c5a1707 100644 --- a/examples/tcp_echo_client.py +++ b/examples/tcp_echo_client.py @@ -1,7 +1,7 @@ import asyncio -async def tcp_echo_client(message, loop): +async def tcp_echo_client(message): reader, writer = await asyncio.open_connection('127.0.0.1', 8888) print('Send: %r' % message) @@ -14,7 +14,11 @@ async def tcp_echo_client(message, loop): writer.close() -message = 'Hello World!' +async def main(): + message = 'Hello World!' + await tcp_echo_client(message) + + loop = asyncio.get_event_loop() -loop.run_until_complete(tcp_echo_client(message, loop)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index f7f58d0..8f0f58b 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -1,5 +1,6 @@ import asyncio + async def handle_echo(reader, writer): data = await reader.read(100) message = data.decode() @@ -13,18 +14,28 @@ async def handle_echo(reader, writer): print("Close the client socket") writer.close() + +async def start_serving(): + server = await asyncio.start_server(handle_echo, '127.0.0.1', 8888) + print('Serving on {}'.format(server.sockets[0].getsockname())) + return stop_serving(server) + + +async def stop_serving(server): + server.close() + await server.wait_closed() + + +# Start the server loop = asyncio.get_event_loop() -coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888) -server = loop.run_until_complete(coro) +stop_coro = loop.run_until_complete(start_serving()) # Serve requests until Ctrl+C is pressed -print('Serving on {}'.format(server.sockets[0].getsockname())) try: loop.run_forever() except KeyboardInterrupt: pass # Close the server -server.close() -loop.run_until_complete(server.wait_closed()) +loop.run_until_complete(stop_coro) loop.close() From 59c32b295f434a2ca95c3d1ee545358fea80990b Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Fri, 18 Nov 2016 16:39:44 +0100 Subject: [PATCH 3/6] Update webscraper examples to promote the use of a main coroutine --- examples/aiohttp_client.py | 47 ++++++++++++---------------- examples/async_client_blocking.py | 38 +++++++++++----------- examples/async_client_nonblocking.py | 41 +++++++++++------------- examples/async_page.py | 11 ++----- examples/simple_server.py | 17 +++------- examples/synchronous_client.py | 29 +++++++---------- 6 files changed, 76 insertions(+), 107 deletions(-) diff --git a/examples/aiohttp_client.py b/examples/aiohttp_client.py index 4ca4da2..6da8223 100644 --- a/examples/aiohttp_client.py +++ b/examples/aiohttp_client.py @@ -1,34 +1,26 @@ -"""aiohttp-based client to retrieve web pages. -""" +"""aiohttp-based client to retrieve web pages.""" -import asyncio -from contextlib import closing import time - +import asyncio import aiohttp async def fetch_page(session, host, port=8000, wait=0): - """Get one page. - """ + """Get one page.""" url = '{}:{}/{}'.format(host, port, wait) with aiohttp.Timeout(10): async with session.get(url) as response: assert response.status == 200 - return await response.text() + text = await response.text() + return text.strip('\n') -def get_multiple_pages(host, waits, port=8000, show_time=True): - """Get multiple pages. - """ - tasks = [] - pages = [] +async def get_multiple_pages(host, waits, port=8000, show_time=True): + """Get multiple pages.""" start = time.perf_counter() - with closing(asyncio.get_event_loop()) as loop: - with aiohttp.ClientSession() as session: - for wait in waits: - tasks.append(fetch_page(session, host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) + with aiohttp.ClientSession() as session: + tasks = [fetch_page(session, host, port, wait) for wait in waits] + pages = await asyncio.gather(*tasks) duration = time.perf_counter() - start sum_waits = sum(waits) if show_time: @@ -37,14 +29,15 @@ def get_multiple_pages(host, waits, port=8000, show_time=True): return pages -if __name__ == '__main__': +async def main(): + """Test it.""" + pages = await get_multiple_pages( + host='http://localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) - def main(): - """Test it. - """ - pages = get_multiple_pages(host='http://localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) - main() +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() diff --git a/examples/async_client_blocking.py b/examples/async_client_blocking.py index 41b244c..0a764e1 100644 --- a/examples/async_client_blocking.py +++ b/examples/async_client_blocking.py @@ -1,23 +1,19 @@ -"""Get "web pages. +"""Get web pages. -Waiting until one pages is download before getting the next." +Waiting until one pages is download before getting the next. """ -import asyncio -from contextlib import closing import time - +import asyncio from async_page import get_page -def get_multiple_pages(host, port, waits, show_time=True): - """Get multiple pages. - """ +async def get_multiple_pages(host, port, waits, show_time=True): + """Get multiple pages.""" start = time.perf_counter() pages = [] - with closing(asyncio.get_event_loop()) as loop: - for wait in waits: - pages.append(loop.run_until_complete(get_page(host, port, wait))) + for wait in waits: + pages.append(await get_page(host, port, wait)) duration = time.perf_counter() - start sum_waits = sum(waits) if show_time: @@ -25,14 +21,16 @@ def get_multiple_pages(host, port, waits, show_time=True): print(msg.format(duration, sum_waits)) return pages -if __name__ == '__main__': - def main(): - """Test it. - """ - pages = get_multiple_pages(host='localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) +async def main(): + """Test it.""" + pages = await get_multiple_pages( + host='localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) + - main() +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() diff --git a/examples/async_client_nonblocking.py b/examples/async_client_nonblocking.py index a62d40d..1d18577 100644 --- a/examples/async_client_nonblocking.py +++ b/examples/async_client_nonblocking.py @@ -1,25 +1,18 @@ -"""Get "web pages. +"""Get web pages. -Waiting until one pages is download before getting the next." +Waiting until one pages is download before getting the next. """ -import asyncio -from contextlib import closing import time - +import asyncio from async_page import get_page -def get_multiple_pages(host, port, waits, show_time=True): - """Get multiple pages. - """ +async def get_multiple_pages(host, port, waits, show_time=True): + """Get multiple pages.""" start = time.perf_counter() - pages = [] - tasks = [] - with closing(asyncio.get_event_loop()) as loop: - for wait in waits: - tasks.append(get_page(host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) + tasks = [get_page(host, port, wait) for wait in waits] + pages = await asyncio.gather(*tasks) duration = time.perf_counter() - start sum_waits = sum(waits) if show_time: @@ -27,14 +20,16 @@ def get_multiple_pages(host, port, waits, show_time=True): print(msg.format(duration, sum_waits)) return pages -if __name__ == '__main__': - def main(): - """Test it. - """ - pages = get_multiple_pages(host='localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) +async def main(): + """Test it.""" + pages = await get_multiple_pages( + host='localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) + - main() +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() diff --git a/examples/async_page.py b/examples/async_page.py index 5f1ba89..03c135e 100644 --- a/examples/async_page.py +++ b/examples/async_page.py @@ -1,7 +1,4 @@ -# file: async_page.py - -"""Get a "web page" asynchronously. -""" +"""Get a web page asynchronously.""" import asyncio @@ -9,8 +6,7 @@ def get_encoding(header): - """Find out encoding. - """ + """Find out encoding.""" for line in header: if line.lstrip().startswith('Content-type'): for entry in line.split(';'): @@ -20,8 +16,7 @@ def get_encoding(header): async def get_page(host, port, wait=0): - """Get a "web page" asynchronously. - """ + """Get a web page asynchronously.""" reader, writer = await asyncio.open_connection(host, port) writer.write(b'\r\n'.join([ 'GET /{} HTTP/1.0'.format(wait).encode(ENCODING), diff --git a/examples/simple_server.py b/examples/simple_server.py index 4a98ed8..1cfca89 100644 --- a/examples/simple_server.py +++ b/examples/simple_server.py @@ -1,7 +1,4 @@ -# file: simple_server.py - -"""Simple HTTP server with GET that waits for given seconds. -""" +"""Simple HTTP server with GET that waits for given seconds.""" from http.server import BaseHTTPRequestHandler, HTTPServer from socketserver import ThreadingMixIn @@ -12,18 +9,15 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - """Simple multi-threaded HTTP server. - """ + """Simple multi-threaded HTTP server.""" pass class MyRequestHandler(BaseHTTPRequestHandler): - """Very simple request handler. Only supports GET. - """ + """Very simple request handler. Only supports GET.""" def do_GET(self): # pylint: disable=invalid-name - """Respond after seconds given in path. - """ + """Respond after seconds given in path.""" try: seconds = float(self.path[1:]) except ValueError: @@ -43,8 +37,7 @@ def do_GET(self): # pylint: disable=invalid-name def run(server_class=ThreadingHTTPServer, handler_class=MyRequestHandler, port=8000): - """Run the simple server on given port. - """ + """Run the simple server on given port.""" server_address = ('', port) httpd = server_class(server_address, handler_class) print('Serving from port {} ...'.format(port)) diff --git a/examples/synchronous_client.py b/examples/synchronous_client.py index bef9576..d62f4f7 100644 --- a/examples/synchronous_client.py +++ b/examples/synchronous_client.py @@ -1,16 +1,13 @@ -"""Synchronous client to retrieve web pages. -""" +"""Synchronous client to retrieve web pages.""" - -from urllib.request import urlopen import time +from urllib.request import urlopen ENCODING = 'ISO-8859-1' def get_encoding(http_response): - """Find out encoding. - """ + """Find out encoding.""" content_type = http_response.getheader('Content-type') for entry in content_type.split(';'): if entry.strip().startswith('charset'): @@ -26,12 +23,11 @@ def get_page(host, port, wait=0): full_url = '{}:{}/{}'.format(host, port, wait) with urlopen(full_url) as http_response: html = http_response.read().decode(get_encoding(http_response)) - return html + return html.strip('\n') def get_multiple_pages(host, port, waits, show_time=True): - """Get multiple pages. - """ + """Get multiple pages.""" start = time.perf_counter() pages = [get_page(host, port, wait) for wait in waits] duration = time.perf_counter() - start @@ -42,14 +38,13 @@ def get_multiple_pages(host, port, waits, show_time=True): return pages -if __name__ == '__main__': +def main(): + """Test it.""" + pages = get_multiple_pages( + host='http://localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) - def main(): - """Test it. - """ - pages = get_multiple_pages(host='http://localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) +if __name__ == '__main__': main() From a1c3996ee57549080f0a0c75e336adea061c2478 Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Fri, 18 Nov 2016 16:53:02 +0100 Subject: [PATCH 4/6] Update webscraper page to match the examples --- webscraper.rst | 90 +++++++++----------------------------------------- 1 file changed, 16 insertions(+), 74 deletions(-) diff --git a/webscraper.rst b/webscraper.rst index df50bcd..9accc14 100644 --- a/webscraper.rst +++ b/webscraper.rst @@ -55,9 +55,7 @@ Let's have a look into the details. This provides a simple multi-threaded web server: .. literalinclude:: examples/simple_server.py - :language: python - :start-after: ENCODING = 'utf-8' - :end-before: class MyRequestHandle + :pyobject: ThreadingHTTPServer It uses multiple inheritance. The mix-in class ``ThreadingMixIn`` provides the multi-threading support and @@ -68,9 +66,7 @@ The request handler only has a ``GET`` method: .. literalinclude:: examples/simple_server.py - :language: python - :start-after: pass - :end-before: def run( + :pyobject: MyRequestHandler It takes the last entry in the paths with ``self.path[1:]``, i.e. our ``2.5``, and tries to convert it into a floating point number. @@ -94,9 +90,7 @@ the encoding specified by ``charset``. This is our helper to find out what the encoding of the page is: .. literalinclude:: examples/synchronous_client.py - :language: python - :start-after: ENCODING = 'ISO-8859-1' - :end-before: def get_page + :pyobject: get_encoding It falls back to ``ISO-8859-1`` if it cannot find a specification of the encoding. @@ -106,16 +100,12 @@ The response is a bytestring and ``.encode()`` is needed to convert it into a string: .. literalinclude:: examples/synchronous_client.py - :language: python - :start-after: return ENCODING - :end-before: def get_multiple_pages + :pyobject: get_page Now, we want multiple pages: .. literalinclude:: examples/synchronous_client.py - :language: python - :start-after: return html - :end-before: if __name__ == '__main__': + :pyobject: get_multiple_pages We just iterate over the waiting times and call ``get_page()`` for all of them. @@ -132,13 +122,10 @@ and get this output:: It took 11.08 seconds for a total waiting time of 11.00. Waited for 1.00 seconds. That's all. - Waited for 5.00 seconds. That's all. - Waited for 3.00 seconds. That's all. - Waited for 2.00 seconds. That's all. @@ -164,16 +151,13 @@ if found. Again, the default encoding is ``ISO-8859-1``: .. literalinclude:: examples/async_page.py - :language: python - :start-after: ENCODING = 'ISO-8859-1' - :end-before: async def get_page + :pyobject: get_encoding The next function is way more interesting because it actually works asynchronously: .. literalinclude:: examples/async_page.py - :language: python - :start-after: return ENCODING + :pyobject: get_page The function ``asyncio.open_connection()`` opens a connection to the given URL. It returns a coroutine. @@ -224,32 +208,7 @@ The interesting things happen in a few lines in ``get_multiple_pages()`` :start-after: pages = [] :end-before: duration -The ``closing`` from the standard library module ``contextlib`` starts -the event loop within a context and closes the loop when leaving the context: - -.. code-block:: python - - with closing(asyncio.get_event_loop()) as loop: - - -The two lines above are equivalent to these five lines: - -.. code-block:: python - - loop = asyncio.get_event_loop(): - try: - - finally: - loop.close() - -We call ``get_page()`` for each page in a loop. -Here we decide to wrap each call in ``loop.run_until_complete()``: - -.. code-block:: python - - for wait in waits: - pages.append(loop.run_until_complete(get_page(host, port, wait))) - +We await ``get_page()`` for each page in a loop. This means, we wait until each pages has been retrieved before asking for the next. Let's run it from the command-line to see what happens:: @@ -283,24 +242,17 @@ waiting for the answer before asking for the next page: The interesting part is in this loop: -.. code-block:: python - - with closing(asyncio.get_event_loop()) as loop: - for wait in waits: - tasks.append(get_page(host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) +.. literalinclude:: examples/async_client_blocking.py + :start-after: start = time.perf_counter() + :end-before: duration We append all return values of ``get_page()`` to our lits of tasks. This allows us to send out all request, in our case four, without waiting for the answers. After sending all of them, we wait for the answers, using: -.. code-block:: python - - loop.run_until_complete(asyncio.gather(*tasks)) + await asyncio.gather(*tasks) -We used ``loop.run_until_complete()`` already for each call to ``get_page()`` -in the previous section. The difference here is the use of ``asyncio.gather()`` that is called with all our tasks in the list ``tasks`` as arguments. The ``asyncio.gather(*tasks)`` means for our example with four list entries: @@ -370,11 +322,8 @@ The whole program looks like this: The function to get one page is asynchronous, because of the ``async def``: - .. literalinclude:: examples/aiohttp_client.py - :language: python - :start-after: import aiohttp - :end-before: def get_multiple_pages + :pyobject: fetch_page The arguments are the same as those for the previous function to retrieve one page plus the additional argument ``session``. @@ -394,13 +343,9 @@ we need to ``await`` again to return the body of the page, using the method This is the interesting part of ``get_multiple_pages()``: -.. code-block:: python - - with closing(asyncio.get_event_loop()) as loop: - with aiohttp.ClientSession() as session: - for wait in waits: - tasks.append(fetch_page(session, host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) +.. literalinclude:: examples/aiohttp_client.py + :start-after: start = time.perf_counter() + :end-before: duration It is very similar to the code in the example of the time-saving implementation with ``asyncio``. @@ -413,13 +358,10 @@ Finally, we run this program:: It took 5.04 seconds for a total waiting time of 11.00. Waited for 1.00 seconds. That's all. - Waited for 5.00 seconds. That's all. - Waited for 3.00 seconds. That's all. - Waited for 2.00 seconds. That's all. From c2a140f6b5b835d877153ac153ae27c724658d93 Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Fri, 18 Nov 2016 16:54:29 +0100 Subject: [PATCH 5/6] Avoid warnings when building the docs --- conf.py | 2 +- getting_started.rst | 37 ++++++++++++++++++-------- glossary.rst | 9 ++++--- hello_world.rst | 7 +++-- webscraper.rst | 65 ++++++++++++++++++++++++++++----------------- 5 files changed, 77 insertions(+), 43 deletions(-) diff --git a/conf.py b/conf.py index 1c98299..409e131 100644 --- a/conf.py +++ b/conf.py @@ -173,7 +173,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/getting_started.rst b/getting_started.rst index af409a6..1af448d 100644 --- a/getting_started.rst +++ b/getting_started.rst @@ -8,9 +8,11 @@ Python 3.5 (or higher) only This documentation is written for Python 3.5 to avail of the new ``async`` and ``await`` keywords. -If you have Python 3.5 installed you only need to install ``aiohttp``:: +If you have Python 3.5 installed you only need to install ``aiohttp``: - pip install -U aiohttp +.. sourcecode:: console + + $ pip install -U aiohttp If you don't have Python 3.5 installed yet, you have several options to install it. @@ -21,27 +23,37 @@ All platforms with ``conda`` * Download and install `Miniconda `_ for our platform. * Create a new Python 3.5 environment (named ``aio35``, use a different - if you like):: + if you like): + + .. sourcecode:: console - conda create -n aio35 python=3.5 + $ conda create -n aio35 python=3.5 * Activate it. - Linux and OS X:: + Linux and OS X: + + .. sourcecode:: console $ source activate aio35 - Windows:: + Windows: + + .. sourcecode:: console $ source activate aio35 -* Install ``aiohttp``:: +* Install ``aiohttp``: + + .. sourcecode:: console $(aio35) pip install aiohttp + Platform specific ----------------- .. would be good to have some word about installing on Windows + * Windows: The easiest way to use Python 3.5 would be to use a package manager such as conda. See the installation instructions above. * Mac OS X: Install `Homebrew `_ + will be done. See the `Future section`_ of the official documentation. task It represents the execution of a coroutine and take care the result in a - future. More details in `official documentation - `_ + future. See the `Task section`_ of the official documentation. + +.. _Future section: https://docs.python.org/3/library/asyncio-task.html#future +.. _Task section: https://docs.python.org/3/library/asyncio-task.html#task diff --git a/hello_world.rst b/hello_world.rst index 6b8500f..8c4dbcf 100644 --- a/hello_world.rst +++ b/hello_world.rst @@ -47,9 +47,12 @@ all scheduled :term:`tasks ` could execute, which results in a warning. .. literalinclude:: examples/loop_stop.py -Warning:: +Output: +.. sourcecode:: console + + second hello + first hello Task was destroyed but it is pending! task: wait_for=> - diff --git a/webscraper.rst b/webscraper.rst index 9accc14..c493b3c 100644 --- a/webscraper.rst +++ b/webscraper.rst @@ -19,28 +19,40 @@ A Mock Web Server This is a very simple web server. (See below for the code.) Its only purpose is to wait for a given amount of time. -Test it by running it from the command line:: +Test it by running it from the command line: + +.. sourcecode:: console $ python simple_server.py -It will answer like this:: +It will answer like this: + +.. sourcecode:: console Serving from port 8000 ... -Now, open a browser and go to this URL:: +Now, open a browser and go to this URL: + +.. sourcecode:: console http://localhost:8000/ -You should see this text in your browser:: +You should see this text in your browser: + +.. sourcecode:: console Waited for 0.00 seconds. -Now, add ``2.5`` to the URL:: +Now, add ``2.5`` to the URL: + +.. sourcecode:: console http://localhost:8000/2.5 After pressing enter, it will take 2.5 seconds until you see this -response:: +response: + +.. sourcecode:: console Waited for 2.50 seconds. @@ -49,7 +61,6 @@ Use different numbers and see how long it takes until the server responds. The full implementation looks like this: .. literalinclude:: examples/simple_server.py - :language: python Let's have a look into the details. This provides a simple multi-threaded web server: @@ -113,11 +124,15 @@ The function ``time.perf_counter()`` provides a time stamp. Taking two time stamps a different points in time and calculating their difference provides the elapsed run time. -Finally, we can run our client:: +Finally, we can run our client: + +.. sourcecode:: console $ python synchronous_client.py -and get this output:: +and get this output: + +.. sourcecode:: console It took 11.08 seconds for a total waiting time of 11.00. Waited for 1.00 seconds. @@ -174,9 +189,7 @@ Therefore, we need to convert our strings in to bytestrings. Next, we read header and message from the reader, which is a ``StreamReader`` instance. We need to iterate over the reader by using a special or loop for -``asyncio``: - -.. code-block:: python +``asyncio``:: async for raw_line in reader: @@ -204,14 +217,15 @@ The interesting things happen in a few lines in ``get_multiple_pages()`` (the rest of this function just measures the run time and displays it): .. literalinclude:: examples/async_client_blocking.py - :language: python :start-after: pages = [] :end-before: duration We await ``get_page()`` for each page in a loop. This means, we wait until each pages has been retrieved before asking for the next. -Let's run it from the command-line to see what happens:: +Let's run it from the command-line to see what happens: + +.. sourcecode:: console $ async_client_blocking.py It took 11.06 seconds for a total waiting time of 11.00. @@ -249,28 +263,26 @@ The interesting part is in this loop: We append all return values of ``get_page()`` to our lits of tasks. This allows us to send out all request, in our case four, without waiting for the answers. -After sending all of them, we wait for the answers, using: +After sending all of them, we wait for the answers, using:: await asyncio.gather(*tasks) The difference here is the use of ``asyncio.gather()`` that is called with all our tasks in the list ``tasks`` as arguments. -The ``asyncio.gather(*tasks)`` means for our example with four list entries: - -.. code-block:: python +The ``asyncio.gather(*tasks)`` means for our example with four list entries:: asyncio.gather(tasks[0], tasks[1], tasks[2], tasks[3]) -So, for a list with 100 tasks it would mean: - -.. code-block:: python +So, for a list with 100 tasks it would mean:: asyncio.gather(tasks[0], tasks[1], tasks[2], # 96 more tasks here tasks[99]) -Let's see if we got any faster:: +Let's see if we got any faster: + +.. sourcecode:: console $ async_client_nonblocking.py It took 5.08 seconds for a total waiting time of 11.00. @@ -309,10 +321,11 @@ High-Level Approach with ``aiohttp`` The library aiohttp_ allows to write HTTP client and server applications, using a high-level approach. -Install with:: +Install with: - $ pip install aiohttp +.. sourcecode:: console + $ pip install aiohttp .. _aiohttp: https://aiohttp.readthedocs.io/en/stable/ @@ -352,7 +365,9 @@ with ``asyncio``. The only difference is the opened client session and handing over this session to ``fetch_page()`` as the first argument. -Finally, we run this program:: +Finally, we run this program: + +.. sourcecode:: console $ python aiohttp_client.py It took 5.04 seconds for a total waiting time of 11.00. From 96d87c5997709e853ef4bdf2a0ffc4ff06a135de Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Tue, 28 Mar 2017 14:56:18 +0200 Subject: [PATCH 6/6] Simplify tcp_echo_server --- examples/tcp_echo_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index 8f0f58b..61da1d1 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -18,7 +18,7 @@ async def handle_echo(reader, writer): async def start_serving(): server = await asyncio.start_server(handle_echo, '127.0.0.1', 8888) print('Serving on {}'.format(server.sockets[0].getsockname())) - return stop_serving(server) + return server async def stop_serving(server): @@ -28,7 +28,7 @@ async def stop_serving(server): # Start the server loop = asyncio.get_event_loop() -stop_coro = loop.run_until_complete(start_serving()) +server = loop.run_until_complete(start_serving()) # Serve requests until Ctrl+C is pressed try: @@ -37,5 +37,5 @@ async def stop_serving(server): pass # Close the server -loop.run_until_complete(stop_coro) +loop.run_until_complete(stop_serving(server)) loop.close()