Python 38


[homer@vision test_numpy]$ module add python/py3.8 # charge le module
[homer@vision test_numpy]$ pip list # montre les modules disponibles pour la version de python
[homer@vision ~]$ pip list
Package                Version
---------------------- ---------
absl-py                0.10.0
argon2-cffi            20.1.0
astunparse             1.6.3
async-generator        1.10
attrs                  20.2.0
backcall               0.2.0
bleach                 3.2.1
bokeh                  2.2.3
cachetools             4.1.1
certifi                2020.6.20
cffi                   1.14.3
chardet                3.0.4
cycler                 0.10.0
Cython                 0.29.21
decorator              4.4.2
defusedxml             0.6.0
entrypoints            0.3
fastrlock              0.5
gast                   0.3.3
google-auth            1.22.1
google-auth-oauthlib   0.4.1
google-pasta           0.2.0
grpcio                 1.33.1
h5py                   2.10.0
idna                   2.10
imagecodecs            2020.5.30
imageio                2.9.0
ipykernel              5.3.4
ipython                7.18.1
ipython-genutils       0.2.0
ipywidgets             7.5.1
jedi                   0.17.2
Jinja2                 2.11.2
joblib                 0.17.0
json5                  0.9.5
jsonschema             3.2.0
jupyter                1.0.0
jupyter-client         6.1.7
jupyter-console        6.2.0
jupyter-core           4.6.3
jupyterlab             2.2.8
jupyterlab-pygments    0.1.2
jupyterlab-server      1.2.0
Keras                  2.4.3
Keras-Preprocessing    1.1.2
kiwisolver             1.2.0
Markdown               3.3.2
MarkupSafe             1.1.1
matplotlib             3.3.2
meshio                 4.3.0
mistune                0.8.4
nb-clean               1.6.0
nbclient               0.5.0
nbconvert              6.0.7
nbformat               5.0.7
nest-asyncio           1.4.1
networkx               2.5
nose                   1.3.7
notebook               6.1.4
numpy                  1.18.5
oauthlib               3.1.0
opt-einsum             3.3.0
packaging              20.4
pandas                 1.1.3
pandocfilters          1.4.2
parso                  0.7.1
pexpect                4.8.0
pickleshare            0.7.5
Pillow                 7.2.0
pip                    20.2.4
pipdeptree             1.0.0
ply                    3.11
prometheus-client      0.8.0
prompt-toolkit         3.0.7
protobuf               3.13.0
ptyprocess             0.6.0
pyasn1                 0.4.8
pyasn1-modules         0.2.8
pycparser              2.20
Pygments               2.7.1
pygmsh                 7.1.0
pyparsing              2.4.7
pyrsistent             0.17.3
python-dateutil        2.8.1
pytz                   2020.1
PyYAML                 5.3.1
pyzmq                  19.0.2
qtconsole              4.7.7
QtPy                   1.9.0
requests               2.24.0
requests-oauthlib      1.3.0
rsa                    4.6
scikit-learn           0.23.2
scipy                  1.5.2
Send2Trash             1.5.0
setuptools             49.2.1
six                    1.15.0
tensorboard            2.3.0
tensorboard-plugin-wit 1.7.0
tensorflow-estimator   2.3.0
tensorflow-gpu         2.3.1
termcolor              1.1.0
terminado              0.9.1
testpath               0.4.4
threadpoolctl          2.1.0
tifffile               2020.10.1
tornado                6.0.4
traitlets              5.0.4
typing-extensions      3.7.4.3
urllib3                1.25.10
vtk                    9.0.1
vtkplotter             2020.3.1
wcwidth                0.2.5
webencodings           0.5.1
Werkzeug               1.0.1
wheel                  0.35.1
widgetsnbextension     3.5.1
wrapt                  1.12.1

Exemple

Voici un exemple de script python :

import numpy as np
import time
N = 6000
M = 10000

k_list = [64, 80, 96, 104, 112, 120, 128, 144, 160, 176, 192, 200, 208, 224, 240, 256, 384]

np.show_config()

for K in k_list:
    a = np.array(np.random.random((M, N)), dtype=np.double, order='C', copy=False)
    b = np.array(np.random.random((N, K)), dtype=np.double, order='C', copy=False)
    A = np.matrix(a, dtype=np.double, copy=False)
    B = np.matrix(b, dtype=np.double, copy=False)

    C = A*B

    start = time.time()

    C = A*B
    C = A*B
    C = A*B
    C = A*B
    C = A*B

    end = time.time()

    tm = (end-start) / 5.0

    print ('{0:4}, {1:9.7}'.format(K, tm))

pour effectuer des calculs en interactif :

[homer@vision ~]$ qsub -I -l select=1:ncpus=24:ompthreads=24  -l walltime=00:30:00
[homer@vision test_numpy]$ module add python/py3.8 # charge le module
[homer@vision test_numpy]$ export MKL_NUM_THREADS=$OMP_NUM_THREADS
[homer@vision test_numpy]$ export OPENBLAS_NUM_THREADS=$OMP_NUM_THREADS
[homer@vision ~]$ python -m threadpoolctl -i numpy scipy.linalg # visualise la configuration des modules et des nombres de threads disponibles
[
  {
    "filepath": "/zfs/softs/python/3.8.6/lib/python3.8/site-packages/numpy.libs/libopenblasp-r0-34a18dc3.3.7.so",
    "prefix": "libopenblas",
    "user_api": "blas",
    "internal_api": "openblas",
    "version": "0.3.7",
    "num_threads": 24,
    "threading_layer": "pthreads" 
  },
  {
    "filepath": "/zfs/softs/python/3.8.6/lib/python3.8/site-packages/scipy.libs/libopenblasp-r0-085ca80a.3.9.so",
    "prefix": "libopenblas",
    "user_api": "blas",
    "internal_api": "openblas",
    "version": "0.3.9",
    "num_threads": 24,
    "threading_layer": "pthreads" 
  }
]
[homer@vision ~]$ python test_numpy.py
blas_mkl_info:
  NOT AVAILABLE
blis_info:
  NOT AVAILABLE
openblas_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/usr/local/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
blas_opt_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/usr/local/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
lapack_mkl_info:
  NOT AVAILABLE
openblas_lapack_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/usr/local/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
lapack_opt_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/usr/local/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
  64, 0.01617227
  80, 0.01820054
  96, 0.02352381
 104, 0.02957225
 112, 0.03000288
 120, 0.02968597
 128, 0.0312314
 144, 0.03028569
 160, 0.03841906
 176, 0.03814478
 192, 0.04426975
 200, 0.05388613
 208, 0.05642014
 224, 0.05577822
 240, 0.05859737
 256, 0.05811205
 384, 0.08938122