optimize scripts

KumaTea · KumaTea · commit ef78f825d854 · 2023-12-03T15:54:42.000+08:00
diff --git a/src/README.md b/src/README.md
@@ -1,5 +1,9 @@
 # SOP Checklist
 
+## Build
+
+## Upload
+
 1. (pypy docker) `python3 copy_whls.py`
 2. (pypy docker) `python3 build_many.py`
 3. (pypy host) `cp -rvf whl /mnt/hdd/Temp/`
diff --git a/src/build.py b/src/build.py
@@ -210,10 +210,10 @@ def build(
         )
         load = get_load()
         while load > load_limit:
-            pbar.write(f'Load: {load} > {load_limit}, waiting...')
+            pbar.write(f'Load: {load:.2f} > {load_limit}, waiting...')
             time.sleep(60)
             load = get_load()
-        pbar.write(f'Load: {load} < {load_limit}, start!')
+        pbar.write(f'Load: {load:.2f} < {load_limit}, start!')
         try:
             pbar.write(f'Now running {command}')
             p = subprocess.Popen(
@@ -255,6 +255,10 @@ def build(
                 pbar.write('Built 100 packages. Uninstall all.')
                 count = 0
                 uninst_all(ver, py_path, plat, pbar)
+            elif 'dependency conflicts' in result + error:
+                pbar.write('Found dependency conflicts! Uninstall all.')
+                count = 0
+                uninst_all(ver, py_path, plat, pbar)
         except KeyboardInterrupt:
             pbar.write('Exiting...')
             pbar.write(f'{success=}')
@@ -269,7 +273,11 @@ def build(
     return success, failed
 
 
-NO_UNINST = ['pip', 'setuptools', 'wheel', 'certifi']
+NO_UNINST = [
+    'pip', 'setuptools', 'wheel',
+    'certifi', 'wrapt',
+    'semantic_version'
+]
 
 
 def uninst_all(ver: str, py_path: str, plat: str = 'win', upper_pbar: tqdm = None):
diff --git a/src/host_cache.py b/src/host_cache.py
@@ -1,49 +1,67 @@
 import os
+import logging
 from flask import Flask, send_from_directory
 
 
+WORKDIR = os.path.expanduser('~')
 if os.name == 'nt':
-    WORKDIR = os.path.expanduser('~')
     HTML_DIR = 'E:/Cache/pypy/html'
     CACHE_DIR = 'AppData/Local/pip/cache/wheels'
 else:
-    WORKDIR = '/home/kuma'
     HTML_DIR = '/tmp/html'
     CACHE_DIR = '.cache/pip/wheels'
 
+app = Flask(__name__)
+
+logging.basicConfig(
+    format='%(asctime)s %(levelname)-8s %(message)s',
+    level=logging.INFO,
+    datefmt='%Y-%m-%d %H:%M:%S')
+
+logger = logging.getLogger('werkzeug')
+logger.setLevel(logging.WARNING)
+
 
 def get_local_whl_list():
     if not os.path.isdir(f'{WORKDIR}/{CACHE_DIR}'):
-        print('No wheels yet!')
+        logger.warning('No wheels yet!')
         return []
 
     whl_list = []
     for root, dirs, files in os.walk(f'{WORKDIR}/{CACHE_DIR}'):
         for file in files:
             if file.endswith('.whl'):
                 whl_list.append((file, root.replace(f'{WORKDIR}/', '')))
-    return whl_list
+    logger.warning(f'Found {len(whl_list)} wheels.')
+    unique_whl_list = []
+    unique_whl_name_list = []
+    for whl in whl_list:
+        if whl[0] not in unique_whl_name_list:
+            unique_whl_name_list.append(whl[0])
+            unique_whl_list.append(whl)
+    logger.warning(f'Found {len(unique_whl_list)} unique wheels.')
+    return unique_whl_list
 
 
 def gen_html():
-    print('Generating HTML...')
+    logger.warning('Generating HTML...')
     whl_list = get_local_whl_list()
-    print(f'Found {len(whl_list)} wheels.')
     whl_list.sort(key=lambda x: x[0])
 
     html_head = '<!DOCTYPE html><html><head><meta charset="utf-8"><title>Wheels</title></head><body>\n'
     html_tail = '</body></html>\n'
     html_body = ''
     for whl in whl_list:
-        html_body += f'<a href="{whl[1]}/{whl[0]}">{whl[0]}</a><br>\n'
+        path = whl[1].replace('\\', '/')
+        name = whl[0]
+        html_body += f'<a href="{path}/{name}">{name}</a><br>\n'
     html = html_head + html_body + html_tail
 
     os.makedirs(HTML_DIR, exist_ok=True)
     with open(f'{HTML_DIR}/index.html', 'w', encoding='utf-8') as f:
         f.write(html)
 
 
-app = Flask(__name__)
 gen_html()
 req_count = 0
 
@@ -55,15 +73,18 @@ def index():
     global req_count
     req_count += 1
     if req_count % 100 == 0:
-        print(f'Requested {req_count} times, regenerating HTML...')
+        logger.warning(f'Requested {req_count} times, regenerating HTML...')
         gen_html()
     return send_from_directory(HTML_DIR, 'index.html')
 
 
 @app.route('/<path:path>')
 def send_whl(path):
+    if not 'whl' in path:
+        logger.error(f'Dismissed: {path}')
+        return '', 404
     filename = path.split('/')[-1]
-    print(f'Requested: {filename}')
+    logger.warning(f'Requested: {filename}')
     return send_from_directory(WORKDIR, path)
 
 
diff --git a/src/init-centos-7.sh b/src/init-centos-7.sh
@@ -88,7 +88,20 @@ cd ..
 rm -rvf libpng-1.6.40 libpng-1.6.40.tar.xz
 
 # others
-sudo dnf -y install postgresql-devel libxml2-devel libxslt-devel unixODBC-devel librdkafka-devel freetds-devel
+sudo dnf -y install postgresql-devel libxml2-devel libxslt-devel unixODBC-devel freetds-devel
+
+# 
+# error "confluent-kafka-python requires librdkafka v2.3.0 or later.
+cd /tmp
+wget https://gh.kmtea.eu/https://github.com/confluentinc/librdkafka/archive/refs/tags/v2.3.0.tar.gz
+tar xvzf v2.3.0.tar.gz
+cd librdkafka-2.3.0
+sudo ln -s /opt/python/cp312-cp312/bin/python3 /usr/bin/python3
+./configure
+make
+sudo make install
+cd ..
+rm -rvf librdkafka-2.3.0 v2.3.0.tar.gz
 
 # hdf5
 # Exception: This version of h5py requires HDF5 >= 1.10.4 (got version (1, 8, 12) from environment variable or library)
diff --git a/src/tools.py b/src/tools.py
@@ -31,7 +31,7 @@ def get_pypy_link(ver: str, plat='win64') -> str:
 
 
 def copy_wheels(dst: str):
-    print(f'Copying wheels for pypy...')
+    logging.info(f'Copying wheels for pypy...')
     pip_cache_dir = get_pip_cache_dir()
     if not os.path.exists(pip_cache_dir):
         return None
@@ -41,7 +41,7 @@ def copy_wheels(dst: str):
         os.makedirs(f'{dst}/{ver}', exist_ok=True)
     os.makedirs(f'{dst}/none', exist_ok=True)
 
-    # find whl file in pip cache
+    logging.info('find whl file in pip cache')
     whl_files = []
     for root, dirs, files in os.walk(pip_cache_dir):
         for file in files:
@@ -51,38 +51,44 @@ def copy_wheels(dst: str):
     with open('../whl/wheels.html', 'r', encoding='utf-8') as f:
         whl_html = f.read()
 
-    # copy whl file to wheel dir
+    logging.info('get new wheels')
     new_whl = []
     for root, file in whl_files:
         if file not in whl_html:
             new_whl.append((root, file))
         else:
             if file in saved_sha256sums:
-                logging.warning(f'{file} already exists in saved_sha256sums')
+                logging.warning(f'Skip: \t{file} already exists in saved_sha256sums')
             else:
                 new_whl.append((root, file))
-                logging.warning(f'{file} already exists in wheels.html')
-                logging.warning(f'You MUST remove old file from releases!!!')
+                logging.warning(f'NEW: \t{file} exists in wheels.html but not in saved_sha256sums')
+                logging.warning(f'\tYou MUST remove old file from releases!!!')
 
+    logging.info('select files to copy')
     pbar = tqdm(new_whl)
     copied_files = []
     for root, file in pbar:
         if file in copied_files:
             continue
         else:
             pbar.set_description(f'Coping {file}')
+            matched = False
             for ver in build_versions:
                 if f'pp{ver.replace(".", "")}-pypy{ver.replace(".", "")}' in file:
                     if not (os.path.isfile(f'{dst}/{ver}/{file}') or os.path.isfile(f'{LINUX_MANY_DIR}/done/{file}')):
                         shutil.copy(f'{root}/{file}', f'{dst}/{ver}/{file}')
                         copied_files.append(file)
+                        matched = True
                         break
             if 'none' in file:
                 if not (os.path.isfile(f'{dst}/none/{file}') or os.path.isfile(f'{LINUX_MANY_DIR}/done/{file}')):
                     shutil.copy(f'{root}/{file}', f'{dst}/none/{file}')
                     copied_files.append(file)
+                    matched = True
+            if not matched:
+                logging.error(f'Unmatched: {file}')
 
-    print(f'Copied {len(copied_files)} wheels')
+    logging.info(f'Copied {len(copied_files)} wheels')
     return copied_files
 
 
