prefork, move modules to global

turns out this actually uses less memory due to less COW

1 files changed, 17 insertions, 10 deletions

diff --git a/syntax-highlighting-server.py b/syntax-highlighting-server.py
index 481c4c0..d650664 100755
--- a/syntax-highlighting-server.py
+++ b/syntax-highlighting-server.py
@@ -1,6 +1,12 @@
 #!/usr/bin/env python3
 
+import asyncio
+import argparse
+import gc
+import os
 import pygments
+from aiohttp import web
+from concurrent.futures import ProcessPoolExecutor
 from pygments import highlight
 from pygments.formatters import HtmlFormatter
 from pygments.lexers import guess_lexer, guess_lexer_for_filename
@@ -25,7 +31,6 @@ def do_highlight(filename, data, formatter):
     ])
 
 def parse_args():
-    import argparse
     parser = argparse.ArgumentParser(description='syntax highlighting server',
             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--host', type=str, default='127.0.0.1',
@@ -36,13 +41,11 @@ def parse_args():
             help='pygments formatting style')
     parser.add_argument('--preload', type=bool, default=True,
             help='preload lexers to reduce fork memory usage')
-    parser.add_argument('--max-workers', type=int, default=0,
+    parser.add_argument('--workers', type=int, default=0,
             help='number of workers, 0 is one per cpu')
     return parser.parse_args()
 
 async def handle_highlight(request):
-    import asyncio
-    from aiohttp import web
     loop = asyncio.get_running_loop()
     text = await request.text()
     result = await loop.run_in_executor(
@@ -51,26 +54,30 @@ async def handle_highlight(request):
     return web.Response(text=result)
 
 def run(args, pool=None):
-    from aiohttp import web
     app = web.Application()
     app['pool'] = pool
     app['formatter'] = HtmlFormatter(style=args.style, nobackground=True)
     app.add_routes([web.post('/highlight', handle_highlight)])
     web.run_app(app, host=args.host, port=args.port)
 
+def noop(*args, **kws):
+    pass
+
 def main():
     args = parse_args()
 
     if args.preload:
         guess_lexer('')
 
-    if args.max_workers == 0:
-        max_workers = None
+    if args.workers == 0:
+        workers = len(os.sched_getaffinity(0))
     else:
-        max_workers = args.max_workers
+        workers = args.workers
 
-    from concurrent.futures import ProcessPoolExecutor
-    with ProcessPoolExecutor(max_workers=max_workers) as pool:
+    with ProcessPoolExecutor(max_workers=workers) as pool:
+        gc.collect()
+        gc.freeze()
+        pool.map(noop, [None] * workers)
         run(args, pool)
 
 if __name__ == '__main__':