Add an option to configure celery workers size. (#2085)

Author: bkyryliuk

superset/cli.py

@@ -144,8 +144,10 @@ def refresh_druid(datasource, merge):
     session.commit()
 
 
-@manager.command
-def worker():
+@manager.option(
+    '-w', '--workers', default=config.get("SUPERSET_CELERY_WORKERS", 32),
+    help="Number of celery server workers to fire up")
+def worker(workers):
     """Starts a Superset worker for async SQL query execution."""
     # celery -A tasks worker --loglevel=info
     print("Starting SQL Celery worker.")
@@ -159,5 +161,6 @@ def worker():
         'broker': config.get('CELERY_CONFIG').BROKER_URL,
         'loglevel': 'INFO',
         'traceback': True,
+        'concurrency': int(workers),
     }
     c_worker.run(**options)

superset/config.py

@@ -32,6 +32,7 @@
 ROW_LIMIT = 50000
 VIZ_ROW_LIMIT = 10000
 SUPERSET_WORKERS = 2
+SUPERSET_CELERY_WORKERS = 32
 
 SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
 SUPERSET_WEBSERVER_PORT = 8088