summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex Xu (Hello71) <alex_y_xu@yahoo.ca>2021-02-07 09:36:33 -0500
committerAlex Xu (Hello71) <alex_y_xu@yahoo.ca>2021-02-07 09:36:33 -0500
commit41bc2a0340d4f99c50d0383e4bb3265840f6e68f (patch)
tree09bc6a48250f38dfb0fa9f57a27c09cf88857389
parent953e7cde90e3220849af329ba5f194b6d14277a4 (diff)
downloadcgit-syntax-highlighting-41bc2a0340d4f99c50d0383e4bb3265840f6e68f.tar.xz
cgit-syntax-highlighting-41bc2a0340d4f99c50d0383e4bb3265840f6e68f.zip
prefork, move modules to global
turns out this actually uses less memory due to less COW
-rwxr-xr-xsyntax-highlighting-server.py27
1 files changed, 17 insertions, 10 deletions
diff --git a/syntax-highlighting-server.py b/syntax-highlighting-server.py
index 481c4c0..d650664 100755
--- a/syntax-highlighting-server.py
+++ b/syntax-highlighting-server.py
@@ -1,6 +1,12 @@
#!/usr/bin/env python3
+import asyncio
+import argparse
+import gc
+import os
import pygments
+from aiohttp import web
+from concurrent.futures import ProcessPoolExecutor
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import guess_lexer, guess_lexer_for_filename
@@ -25,7 +31,6 @@ def do_highlight(filename, data, formatter):
])
def parse_args():
- import argparse
parser = argparse.ArgumentParser(description='syntax highlighting server',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--host', type=str, default='127.0.0.1',
@@ -36,13 +41,11 @@ def parse_args():
help='pygments formatting style')
parser.add_argument('--preload', type=bool, default=True,
help='preload lexers to reduce fork memory usage')
- parser.add_argument('--max-workers', type=int, default=0,
+ parser.add_argument('--workers', type=int, default=0,
help='number of workers, 0 is one per cpu')
return parser.parse_args()
async def handle_highlight(request):
- import asyncio
- from aiohttp import web
loop = asyncio.get_running_loop()
text = await request.text()
result = await loop.run_in_executor(
@@ -51,26 +54,30 @@ async def handle_highlight(request):
return web.Response(text=result)
def run(args, pool=None):
- from aiohttp import web
app = web.Application()
app['pool'] = pool
app['formatter'] = HtmlFormatter(style=args.style, nobackground=True)
app.add_routes([web.post('/highlight', handle_highlight)])
web.run_app(app, host=args.host, port=args.port)
+def noop(*args, **kws):
+ pass
+
def main():
args = parse_args()
if args.preload:
guess_lexer('')
- if args.max_workers == 0:
- max_workers = None
+ if args.workers == 0:
+ workers = len(os.sched_getaffinity(0))
else:
- max_workers = args.max_workers
+ workers = args.workers
- from concurrent.futures import ProcessPoolExecutor
- with ProcessPoolExecutor(max_workers=max_workers) as pool:
+ with ProcessPoolExecutor(max_workers=workers) as pool:
+ gc.collect()
+ gc.freeze()
+ pool.map(noop, [None] * workers)
run(args, pool)
if __name__ == '__main__':