diff options
| -rwxr-xr-x | syntax-highlighting-server.py | 27 |
1 files changed, 17 insertions, 10 deletions
diff --git a/syntax-highlighting-server.py b/syntax-highlighting-server.py index 481c4c0..d650664 100755 --- a/syntax-highlighting-server.py +++ b/syntax-highlighting-server.py @@ -1,6 +1,12 @@ #!/usr/bin/env python3 +import asyncio +import argparse +import gc +import os import pygments +from aiohttp import web +from concurrent.futures import ProcessPoolExecutor from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers import guess_lexer, guess_lexer_for_filename @@ -25,7 +31,6 @@ def do_highlight(filename, data, formatter): ]) def parse_args(): - import argparse parser = argparse.ArgumentParser(description='syntax highlighting server', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--host', type=str, default='127.0.0.1', @@ -36,13 +41,11 @@ def parse_args(): help='pygments formatting style') parser.add_argument('--preload', type=bool, default=True, help='preload lexers to reduce fork memory usage') - parser.add_argument('--max-workers', type=int, default=0, + parser.add_argument('--workers', type=int, default=0, help='number of workers, 0 is one per cpu') return parser.parse_args() async def handle_highlight(request): - import asyncio - from aiohttp import web loop = asyncio.get_running_loop() text = await request.text() result = await loop.run_in_executor( @@ -51,26 +54,30 @@ async def handle_highlight(request): return web.Response(text=result) def run(args, pool=None): - from aiohttp import web app = web.Application() app['pool'] = pool app['formatter'] = HtmlFormatter(style=args.style, nobackground=True) app.add_routes([web.post('/highlight', handle_highlight)]) web.run_app(app, host=args.host, port=args.port) +def noop(*args, **kws): + pass + def main(): args = parse_args() if args.preload: guess_lexer('') - if args.max_workers == 0: - max_workers = None + if args.workers == 0: + workers = len(os.sched_getaffinity(0)) else: - max_workers = args.max_workers + workers = args.workers - from concurrent.futures import ProcessPoolExecutor - with ProcessPoolExecutor(max_workers=max_workers) as pool: + with ProcessPoolExecutor(max_workers=workers) as pool: + gc.collect() + gc.freeze() + pool.map(noop, [None] * workers) run(args, pool) if __name__ == '__main__': |