Skip to content

Commit 7ee661b

Browse files
johnleesjohnlees
authored
Attempt to fix for clang 15 (#93)
* Update Xoshiro return type * Prints to help debug in makefile * Try and resolve hdf5 version issue * Pin h5py version in tests * Pin exact versions * Revert the hdf5 installation * Revert original change to check for segfault * Move the testing to gha * Try to get segfault backtrace * Change library location * Attempt to add ssh to gha * Try ssh again * Fix typo in yaml * Change install command * Use single threaded tests * Revert changes in the wrapper * Add tests back to the azure version * Try adding libgomp via mamba, single multicore test * Another interactive debugging run * Forgot to remove delete line * Check for keyboard interrupt only on main thread * Add test for openmp dists; try to fix dist segfault * Try restricting signal check to thread 0
1 parent 06cc248 commit 7ee661b

File tree

9 files changed

+96
-35
lines changed

9 files changed

+96
-35
lines changed

‎.github/workflows/azure_ci.yml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Python package
2+
# Create and test a Python package on multiple Python versions.
3+
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
4+
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
5+
6+
name: Run tests
7+
8+
on: [push]
9+
10+
jobs:
11+
test:
12+
13+
runs-on: ubuntu-latest
14+
strategy:
15+
matrix:
16+
python-version: [3.8]
17+
18+
steps:
19+
- uses: actions/checkout@v2
20+
- name: Set up Python ${{ matrix.python-version }}
21+
uses: actions/setup-python@v2
22+
with:
23+
python-version: ${{ matrix.python-version }}
24+
- name: Get current date
25+
id: date
26+
run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}"
27+
- name: Install Conda environment from environment.yml
28+
uses: mamba-org/setup-micromamba@v1
29+
with:
30+
micromamba-version: '1.4.6-0'
31+
environment-file: environment.yml
32+
# persist on the same day.
33+
cache-environment-key: environment-${{ steps.date.outputs.date }}
34+
cache-downloads-key: downloads-${{ steps.date.outputs.date }}
35+
- name: Install and run_test.py
36+
shell: bash -l {0}
37+
run: |
38+
python -m pip install --no-deps --ignore-installed . -vvv
39+
cd test && python run_test.py
40+
41+
## For debugging
42+
- name: Install debug version
43+
if: failure()
44+
shell: bash -l {0}
45+
run: |
46+
python setup.py build_ext --debug install
47+
- name: Setup tmate session for interactive debugging
48+
if: failure()
49+
uses: mxschmitt/action-tmate@v3
50+
with:
51+
limit-access-to-actor: true
52+
detached: true

‎CMakeLists.txt

+7-6
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,13 @@ if(DEFINED ENV{CONDA_PREFIX})
4545
include_directories($ENV{CONDA_PREFIX}/include)
4646
link_directories($ENV{CONDA_PREFIX}/lib)
4747
link_directories($ENV{CONDA_PREFIX}/lib/intel64)
48+
else()
49+
set(HDF5_FIND_DEBUG TRUE)
50+
find_package(HDF5 REQUIRED COMPONENTS CXX)
51+
include_directories(${HDF5_INCLUDE_DIRS})
4852
endif()
4953

5054
# Add libraries
51-
52-
find_package(HDF5 REQUIRED COMPONENTS CXX)
53-
include_directories(${HDF5_INCLUDE_DIRS})
5455
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/vendor/highfive/include)
5556
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
5657

@@ -148,11 +149,11 @@ if(CMAKE_CUDA_COMPILER)
148149
#set_property(TARGET "${TARGET_NAME}" PROPERTY CUDA_ARCHITECTURES OFF)
149150
endif()
150151
target_link_libraries("${TARGET_NAME}" PRIVATE pybind11::module Eigen3::Eigen
151-
${HDF5_LIBRARIES} ${BLAS_LIBRARIES} gfortran m dl)
152+
${BLAS_LIBRARIES} gfortran m dl)
152153
if(DEFINED ENV{CONDA_PREFIX} AND (NOT APPLE OR CMAKE_COMPILER_IS_GNUCC OR ENV{SKETCHLIB_INSTALL} EQUAL "conda"))
153-
target_link_libraries("${TARGET_NAME}" PRIVATE gomp z)
154+
target_link_libraries("${TARGET_NAME}" PRIVATE hdf5_cpp hdf5 gomp z)
154155
else()
155-
target_link_libraries("${TARGET_NAME}" PRIVATE ZLIB::ZLIB)
156+
target_link_libraries("${TARGET_NAME}" PRIVATE ${HDF5_LIBRARIES} ZLIB::ZLIB)
156157
find_package(OpenMP)
157158
if(OpenMP_CXX_FOUND)
158159
target_link_libraries("${TARGET_NAME}" PRIVATE OpenMP::OpenMP_CXX)

‎azure-pipelines.yml

+1-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
# Python package
2-
# Create and test a Python package on multiple Python versions.
3-
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
4-
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
1+
# This just checks the package can be installed using CUDA, no testing
52

63
trigger:
74
- master

‎environment.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ dependencies:
1717
- highfive
1818
- hdf5
1919
- h5py
20-
- nlohmann_json
20+
- libgomp
2121
- openblas
2222
- libgfortran-ng
2323
- nvcc_linux-64

‎pp_sketch/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33

44
'''PopPUNK sketching functions'''
55

6-
__version__ = '2.1.1'
6+
__version__ = '2.1.2'

‎src/Makefile

+4
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ install_python: python
116116
install $(PYTHON_LIB) $(PYTHON_LIB_PATH)
117117

118118
gpu/dist.cu.o:
119+
echo ${CUDAFLAGS}
120+
echo ${CPPFLAGS}
121+
echo ${CXXFLAGS}
122+
echo ${CFLAGS}
119123
nvcc $(CUDAFLAGS) $(CPPFLAGS) -DGPU_AVAILABLE -x cu -c gpu/dist.cu -o $@
120124

121125
gpu/sketch.cu.o:

‎src/api.cpp

+16-12
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,7 @@ std::vector<Reference> create_sketches(
8181
std::vector<std::runtime_error> errors;
8282
#pragma omp parallel for schedule(dynamic, 5) num_threads(num_threads)
8383
for (unsigned int i = 0; i < names.size(); i++) {
84-
if (interrupt || PyErr_CheckSignals() != 0) {
85-
interrupt = true;
86-
} else {
84+
if (!interrupt) {
8785
try {
8886
SeqBuf seq_in(files[i], kmer_lengths.back());
8987
sketches[i] = Reference(names[i], seq_in, kmer_seeds, sketchsize64,
@@ -101,6 +99,9 @@ std::vector<Reference> create_sketches(
10199

102100
if (omp_get_thread_num() == 0) {
103101
sketch_progress.tick_count(done_count);
102+
if (PyErr_CheckSignals() != 0) {
103+
interrupt = true;
104+
}
104105
}
105106
}
106107
sketch_progress.finalise();
@@ -198,9 +199,7 @@ NumpyMatrix query_db(std::vector<Reference> &ref_sketches,
198199
// Iterate upper triangle
199200
#pragma omp parallel for schedule(dynamic, 5) num_threads(num_threads) shared(progress)
200201
for (size_t i = 0; i < ref_sketches.size(); i++) {
201-
if (interrupt || PyErr_CheckSignals() != 0) {
202-
interrupt = true;
203-
} else {
202+
if (!interrupt) {
204203
for (size_t j = i + 1; j < ref_sketches.size(); j++) {
205204
size_t pos = square_to_condensed(i, j, ref_sketches.size());
206205
if (jaccard) {
@@ -219,6 +218,9 @@ NumpyMatrix query_db(std::vector<Reference> &ref_sketches,
219218
{
220219
progress += MAX(1, n_progress_ticks / dist_rows);
221220
dist_progress.tick_count(progress);
221+
if (omp_get_thread_num() == 0 && PyErr_CheckSignals() != 0) {
222+
interrupt = true;
223+
}
222224
}
223225
}
224226
}
@@ -244,9 +246,7 @@ NumpyMatrix query_db(std::vector<Reference> &ref_sketches,
244246
#pragma omp parallel for collapse(2) schedule(static) num_threads(num_threads)
245247
for (unsigned int q_idx = 0; q_idx < query_sketches.size(); q_idx++) {
246248
for (unsigned int r_idx = 0; r_idx < ref_sketches.size(); r_idx++) {
247-
if (interrupt || PyErr_CheckSignals() != 0) {
248-
interrupt = true;
249-
} else {
249+
if (!interrupt) {
250250
const long dist_row = q_idx * ref_sketches.size() + r_idx;
251251
if (jaccard) {
252252
for (unsigned int kmer_idx = 0; kmer_idx < kmer_lengths.size();
@@ -270,6 +270,9 @@ NumpyMatrix query_db(std::vector<Reference> &ref_sketches,
270270
{
271271
progress += MAX(1, n_progress_ticks / dist_rows);
272272
dist_progress.tick_count(progress);
273+
if (omp_get_thread_num() == 0 && PyErr_CheckSignals() != 0) {
274+
interrupt = true;
275+
}
273276
}
274277
}
275278
}
@@ -342,9 +345,7 @@ sparse_coo query_db_sparse(std::vector<Reference> &ref_sketches,
342345
#pragma omp parallel for schedule(static) num_threads(num_threads) shared(progress)
343346
for (size_t i = 0; i < ref_sketches.size(); i++) {
344347
std::vector<float> row_dists(ref_sketches.size());
345-
if (interrupt || PyErr_CheckSignals() != 0) {
346-
interrupt = true;
347-
} else {
348+
if (!interrupt) {
348349
for (size_t j = 0; j < ref_sketches.size(); j++) {
349350
if (i != j) {
350351
if (jaccard) {
@@ -370,6 +371,9 @@ sparse_coo query_db_sparse(std::vector<Reference> &ref_sketches,
370371
{
371372
progress += MAX(1, n_progress_ticks / dist_rows);
372373
dist_progress.tick_count(progress);
374+
if (omp_get_thread_num() == 0 && PyErr_CheckSignals() != 0) {
375+
interrupt = true;
376+
}
373377
}
374378
}
375379
long offset = i * kNN;

‎src/random/rng.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class Xoshiro
1616
{
1717
public:
1818
// Definitions to be used as URNG in C++11
19-
typedef size_t result_type;
19+
typedef uint64_t result_type;
2020
static constexpr size_t min() { return std::numeric_limits<uint64_t>::min(); }
2121
static constexpr size_t max() { return std::numeric_limits<uint64_t>::max(); }
2222
uint64_t operator()(); // generate random number U(min, max)

‎test/run_test.py

+13-10
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,20 @@
2626

2727
# create sketches
2828
sys.stderr.write("Sketch smoke test\n")
29+
subprocess.run("python ../sketchlib-runner.py sketch -l references.txt -o test_db -s 10000 -k 15,29,4 --cpus 1", shell=True, check=True)
30+
os.remove("test_db.h5")
2931
subprocess.run("python ../sketchlib-runner.py sketch -l references.txt -o test_db -s 10000 -k 15,29,4 --cpus 2", shell=True, check=True)
30-
subprocess.run("python ../sketchlib-runner.py sketch -l references.txt -o test_db_phased --codon-phased --cpus 2", shell=True, check=True)
32+
subprocess.run("python ../sketchlib-runner.py sketch -l references.txt -o test_db_phased --codon-phased --cpus 1", shell=True, check=True)
3133
subprocess.run("python ../sketchlib-runner.py sketch 12673_8#24.contigs_velvet.fa 12673_8#34.contigs_velvet.fa -o test_db_small -s 1000 --kmer 14", shell=True, check=True)
3234
subprocess.run("python ../sketchlib-runner.py add random test_db --cpus 2", shell=True, check=True)
3335
# calculate distances
3436
sys.stderr.write("Distance integration test\n")
35-
subprocess.run("python ../sketchlib-runner.py query dist test_db --cpus 2", shell=True, check=True) # checks if can be run
36-
subprocess.run("python ../sketchlib-runner.py query dist test_db -o ppsketch --cpus 2", shell=True, check=True) # checks if can be run
37-
subprocess.run("python ../sketchlib-runner.py query jaccard test_db_small --cpus 2", shell=True, check=True) # checks if can be run
37+
subprocess.run("python ../sketchlib-runner.py query dist test_db --cpus 1", shell=True, check=True) # checks if can be run
38+
subprocess.run("python ../sketchlib-runner.py query dist test_db --cpus 2", shell=True, check=True) # checks if can be run w/ openmp
39+
subprocess.run("python ../sketchlib-runner.py query dist test_db -o ppsketch --cpus 1", shell=True, check=True) # checks if can be run
40+
subprocess.run("python ../sketchlib-runner.py query jaccard test_db_small --cpus 1", shell=True, check=True) # checks if can be run
3841
subprocess.run("python test-dists.py --ref-db test_db --results ppsketch_ref", shell=True, check=True) # checks results match
39-
subprocess.run("python ../sketchlib-runner.py query dist test_db_phased --cpus 2", shell=True, check=True) # checks if can be run
42+
subprocess.run("python ../sketchlib-runner.py query dist test_db_phased --cpus 1", shell=True, check=True) # checks if can be run
4043
subprocess.run("python test-dists.py --ref-db test_db_phased --results ppsketch_ref_phased", shell=True, check=True) # checks results match
4144

4245
sys.stderr.write("Sparse distance smoke test\n")
@@ -46,19 +49,19 @@
4649
subprocess.run("python ../sketchlib-runner.py query sparse jaccard test_db --kNN 2 --kmer 19", shell=True, check=True) # checks if can be run
4750

4851
sys.stderr.write("Ref v query distance smoke test\n")
49-
subprocess.run("python ../sketchlib-runner.py sketch -l rlist.txt -o r_db --cpus 2", shell=True, check=True)
50-
subprocess.run("python ../sketchlib-runner.py sketch -l qlist.txt -o q_db --cpus 2", shell=True, check=True)
52+
subprocess.run("python ../sketchlib-runner.py sketch -l rlist.txt -o r_db --cpus 1", shell=True, check=True)
53+
subprocess.run("python ../sketchlib-runner.py sketch -l qlist.txt -o q_db --cpus 1", shell=True, check=True)
5154
subprocess.run("python ../sketchlib-runner.py query dist r_db q_db.h5", shell=True, check=True) # checks if can be run
5255
subprocess.run("python ../sketchlib-runner.py query jaccard r_db q_db", shell=True, check=True) # checks if can be run
5356

5457
# Joining
5558
sys.stderr.write("Join smoke test\n")
56-
subprocess.run("python ../sketchlib-runner.py sketch -l db1_refs.txt -o db1 --cpus 2", shell=True, check=True)
57-
subprocess.run("python ../sketchlib-runner.py sketch -l db2_refs.txt -o db2 --cpus 2", shell=True, check=True)
59+
subprocess.run("python ../sketchlib-runner.py sketch -l db1_refs.txt -o db1 --cpus 1", shell=True, check=True)
60+
subprocess.run("python ../sketchlib-runner.py sketch -l db2_refs.txt -o db2 --cpus 1", shell=True, check=True)
5861
subprocess.run("python ../sketchlib-runner.py join db1.h5 db2.h5 -o joined", shell=True, check=True)
5962
# Random
6063
sys.stderr.write("Random test\n")
61-
subprocess.run("python ../sketchlib-runner.py remove random test_db --cpus 2", shell=True, check=True)
64+
subprocess.run("python ../sketchlib-runner.py remove random test_db --cpus 1", shell=True, check=True)
6265
# Matrix
6366
sys.stderr.write("Matrix integration test\n")
6467
subprocess.run("python test-matrix.py", shell=True, check=True)

0 commit comments

Comments
 (0)