From f09213eb33d047859dc11a0b1381ba4f7201360b Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Thu, 16 Jul 2020 16:08:57 -0700 Subject: [PATCH] various updates from ARIA-related projects (#149) * add Dockerfile and SConfigISCE to build cuda-enabled modules * add Dockerfile and SConfigISCE to build cuda-enabled modules * set correct name * update for centos7 and cuda-dev image * restore * disable GPUampcor * update * save * sync up Dockerfile updates * fix base image * change docker image names * create gpu vs. cpu specific tags * update with nodata WBD Stitcher * fix segmentation fault when running sciflo workflows - remove dependencies that downgrade the `geos` conda package * sync fix for segmentation fault to cuda builds * ping ffi to version 1.12.2 Latest version of ffi, 1.13.0, breaks installation of fpm: ``` $ sudo gem install --no-ri --no-rdoc fpm Fetching: cabin-0.9.0.gem (100%) Successfully installed cabin-0.9.0 Fetching: backports-3.17.2.gem (100%) Successfully installed backports-3.17.2 Fetching: arr-pm-0.0.10.gem (100%) Successfully installed arr-pm-0.0.10 Fetching: clamp-1.0.1.gem (100%) Successfully installed clamp-1.0.1 Fetching: ffi-1.13.0.gem (100%) ERROR: Error installing fpm: ffi requires Ruby version >= 2.3. ``` TODO: remove pin when ffi's backwards compatiblity is restored. * comment out unused import * add build circleci job to PR tests * pin ffi in cuda build * revert logging hackery * fix "target_include_directories called with non-compilable target type" errors * Keep source directory for topsStack Co-authored-by: dustinlo Co-authored-by: shitong01 Co-authored-by: Ryan Burns <47790121+rtburns-jpl@users.noreply.github.com> --- .circleci/config.yml | 5 +- .../zerodop/GPUtopozero/src/AkimaLib.cpp | 3 - contrib/PyCuAmpcor/src/Makefile | 2 +- contrib/demUtils/swbdstitcher/SWBDStitcher.py | 13 +- contrib/stack/topsStack/grossOffsets.py | 1 + docker/Dockerfile | 40 +++--- docker/Dockerfile.cuda | 117 ++++++++++++++++++ docker/README.md | 6 +- docker/SConfigISCE.cuda | 41 ++++++ docker/build_env.sh | 6 +- scons_tools/cuda.py | 2 +- 11 files changed, 201 insertions(+), 35 deletions(-) create mode 100644 docker/Dockerfile.cuda create mode 100644 docker/SConfigISCE.cuda diff --git a/.circleci/config.yml b/.circleci/config.yml index c154bf5..7a7c776 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: pwd mkdir config build install . /opt/conda/bin/activate root - conda install -y cython gdal h5py libgdal pytest numpy fftw scipy basemap scons opencv hdf4 hdf5 netcdf4 libgcc libstdcxx-ng cmake astropy + conda install -y cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 netcdf4 libgcc libstdcxx-ng cmake astropy yum install -y x11-devel motif-devel jq gcc-gfortran - run: @@ -77,7 +77,7 @@ jobs: pwd mkdir config build install . /opt/conda/bin/activate root - conda install --yes cython gdal h5py libgdal pytest numpy fftw scipy basemap scons opencv hdf4 hdf5 netcdf4 libgcc libstdcxx-ng cmake astropy + conda install --yes cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 netcdf4 libgcc libstdcxx-ng cmake astropy yum install -y uuid-devel x11-devel motif-devel jq gcc-gfortran ln -s /opt/conda/bin/cython /opt/conda/bin/cython3 cd /opt/conda/lib @@ -241,6 +241,7 @@ workflows: jobs: - test - test-cmake + - build build-deploy: jobs: - build: diff --git a/components/zerodop/GPUtopozero/src/AkimaLib.cpp b/components/zerodop/GPUtopozero/src/AkimaLib.cpp index a2cea96..2a4b0e2 100644 --- a/components/zerodop/GPUtopozero/src/AkimaLib.cpp +++ b/components/zerodop/GPUtopozero/src/AkimaLib.cpp @@ -16,9 +16,6 @@ using std::max; using std::min; using std::vector; -#if __cplusplus >= 201103L -using std::isnan; -#endif bool AkimaLib::aki_almostEqual(double x, double y) { bool ret = (abs(x - y) <= AKI_EPS) ? true : false; // Compressed version is a little cleaner diff --git a/contrib/PyCuAmpcor/src/Makefile b/contrib/PyCuAmpcor/src/Makefile index b789a59..d431c90 100644 --- a/contrib/PyCuAmpcor/src/Makefile +++ b/contrib/PyCuAmpcor/src/Makefile @@ -2,7 +2,7 @@ PROJECT = CUAMPCOR LDFLAGS = -lcuda -lcudart -lcufft -lcublas CXXFLAGS = -std=c++11 -fpermissive -fPIC -shared -NVCCFLAGS = -ccbin g++ -m64 \ +NVCCFLAGS = -std=c++11 -ccbin g++ -m64 \ -gencode arch=compute_35,code=sm_35 \ -gencode arch=compute_60,code=sm_60 \ -Xcompiler -fPIC -shared -Wno-deprecated-gpu-targets \ diff --git a/contrib/demUtils/swbdstitcher/SWBDStitcher.py b/contrib/demUtils/swbdstitcher/SWBDStitcher.py index e3839fd..efbbace 100755 --- a/contrib/demUtils/swbdstitcher/SWBDStitcher.py +++ b/contrib/demUtils/swbdstitcher/SWBDStitcher.py @@ -52,6 +52,12 @@ URL = Component.Parameter('_url', mandatory = False, doc = "Url for the high resolution water body mask") +NODATA = Component.Parameter('_nodata', + public_name = 'nodata',default = 0, + type = int, + mandatory = False, + doc = "Nodata value for missing tiles") + KEEP_WBDS = Component.Parameter('_keepWbds', public_name='keepWbds', default = False, @@ -220,7 +226,7 @@ class SWBDStitcher(DemStitcher): syntTileCreated = False #check and send a warning if the full region is not available if not self._succeded in self._downloadReport.values(): - self.logger.warning('The full region of interested is not available. Missing region is assumed to be land') + self.logger.warning('The full region of interested is not available. Missing region is assumed to be %s'%(str(self._nodata))) for k,v in self._downloadReport.items(): if v == self._failed:#symlink each missing file to the reference one created in createFillingFile if not syntTileCreated:#create the synthetic Tile the first time around @@ -231,7 +237,7 @@ class SWBDStitcher(DemStitcher): if unzip: mmap = np.memmap(outname,np.int8,'w+',shape=(nLat*tileSize,nLon*tileSize)) - mmap[:,:] = 0 + mmap[:,:] = self._nodata decompressedList = [] pos = 0 for i in range(nLat): @@ -290,7 +296,8 @@ class SWBDStitcher(DemStitcher): parameter_list = ( URL, - KEEP_WBDS + KEEP_WBDS, + NODATA, ) family = 'swbdstitcher' diff --git a/contrib/stack/topsStack/grossOffsets.py b/contrib/stack/topsStack/grossOffsets.py index 0247049..d788a47 100755 --- a/contrib/stack/topsStack/grossOffsets.py +++ b/contrib/stack/topsStack/grossOffsets.py @@ -11,6 +11,7 @@ import isceobj from iscesys.Component.ProductManager import ProductManager as PM import numpy as np from netCDF4 import Dataset +#from mpl_toolkits.basemap import Basemap import gdal from scipy.interpolate import interp2d, griddata diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b9c0a8..daa73f6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM hysds/pge-base:latest +FROM hysds/dev:latest # Set an encoding to make things work smoothly. ENV LANG en_US.UTF-8 @@ -15,6 +15,7 @@ RUN set -ex \ && yum groupinstall -y "development tools" \ && yum install -y \ make ruby-devel rpm-build rubygems \ + && gem install ffi -v 1.12.2 \ && gem install --no-ri --no-rdoc fpm # install isce requirements @@ -30,9 +31,7 @@ RUN set -ex \ numpy \ fftw \ scipy \ - basemap \ scons \ - opencv \ hdf4 \ hdf5 \ netcdf4 \ @@ -40,9 +39,10 @@ RUN set -ex \ libstdcxx-ng \ cmake \ && yum install -y uuid-devel x11-devel motif-devel jq \ - && ln -sf /opt/conda/bin/cython /opt/conda/bin/cython3 + && ln -sf /opt/conda/bin/cython /opt/conda/bin/cython3 \ + && mkdir -p /opt/isce2/src -# link system libuuid +# override system libuuid into conda env to link in libXm and libXt RUN set -ex \ && cd /opt/conda/lib \ && unlink libuuid.so \ @@ -57,26 +57,26 @@ RUN set -ex \ && ( test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so ) # copy repo -COPY . /root/isce2 +COPY . /opt/isce2/src/isce2 # build ISCE RUN set -ex \ && . /opt/conda/bin/activate root \ - && cd /root \ - && source isce2/docker/build_env.sh \ + && cd /opt/isce2/src/isce2 \ + && source docker/build_env.sh \ && mkdir -p $BUILD_DIR \ - && cp isce2/docker/SConfigISCE isce2/configuration/ \ - && cd isce2 \ + && cp docker/SConfigISCE configuration/SConfigISCE \ && scons install \ && cp docker/isce_env.sh $ISCE_INSTALL_ROOT \ - && mkdir -p rpm-build/opt \ - && mv $ISCE_INSTALL_ROOT rpm-build/opt \ + && cd /tmp \ + && mkdir -p /tmp/rpm-build/opt \ + && mv $ISCE_INSTALL_ROOT /tmp/rpm-build/opt \ && curl -s https://api.github.com/repos/$ISCE_ORG/isce2/git/refs/heads/master \ - > rpm-build/opt/isce2/version.json \ - && hash=$(cat rpm-build/opt/isce2/version.json | jq -r .object.sha) \ + > /tmp/rpm-build/opt/isce2/version.json \ + && hash=$(cat /tmp/rpm-build/opt/isce2/version.json | jq -r .object.sha) \ && short_hash=$(echo $hash | cut -c1-5) \ - && fpm -s dir -t rpm -C rpm-build --name isce \ - --prefix=/ --version=2.0 --provides=isce \ + && fpm -s dir -t rpm -C /tmp/rpm-build --name isce \ + --prefix=/ --version=2.3 --provides=isce \ --maintainer=piyush.agram@jpl.nasa.gov \ --description="InSAR Scientific Computing Environment v2 (${hash})" @@ -86,7 +86,7 @@ FROM hysds/pge-base:latest ENV LANG en_US.UTF-8 # install ISCE from RPM -COPY --from=0 /root/isce2/isce-2.0-1.x86_64.rpm /tmp/isce-2.0-1.x86_64.rpm +COPY --from=0 /tmp/isce-2.3-1.x86_64.rpm /tmp/isce-2.3-1.x86_64.rpm # install isce and its minimal requirements RUN set -ex \ @@ -98,8 +98,6 @@ RUN set -ex \ numpy \ fftw \ scipy \ - basemap \ - opencv \ hdf4 \ hdf5 \ netcdf4 \ @@ -112,7 +110,7 @@ RUN set -ex \ && sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1 \ && cd /lib64 \ && ( test -f libgfortran.so || sudo ln -sv libgfortran.so.*.* libgfortran.so ) \ - && sudo yum install -y /tmp/isce-2.0-1.x86_64.rpm \ + && sudo yum install -y /tmp/isce-2.3-1.x86_64.rpm \ && sudo yum clean all \ && sudo rm -rf /var/cache/yum \ - && sudo rm /tmp/isce-2.0-1.x86_64.rpm + && sudo rm /tmp/isce-2.3-1.x86_64.rpm diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda new file mode 100644 index 0000000..250f11e --- /dev/null +++ b/docker/Dockerfile.cuda @@ -0,0 +1,117 @@ +FROM hysds/cuda-dev:latest + +# Set an encoding to make things work smoothly. +ENV LANG en_US.UTF-8 + +# Set ISCE repo +ENV ISCE_ORG isce-framework + +# set to root user +USER root + +# install tools for RPM generation +RUN set -ex \ + && yum update -y \ + && yum groupinstall -y "development tools" \ + && yum install -y \ + make ruby-devel rpm-build rubygems \ + && gem install ffi -v 1.12.2 \ + && gem install --no-ri --no-rdoc fpm + +# install isce requirements +RUN set -ex \ + && . /opt/conda/bin/activate root \ + && conda install --yes \ + cython \ + gdal \ + git \ + h5py \ + libgdal \ + pytest \ + numpy \ + fftw \ + scipy \ + scons \ + hdf4 \ + hdf5 \ + netcdf4 \ + libgcc \ + libstdcxx-ng \ + cmake \ + && yum install -y uuid-devel x11-devel motif-devel jq \ + && ln -sf /opt/conda/bin/cython /opt/conda/bin/cython3 \ + && mkdir -p /opt/isce2/src + +# override system libuuid into conda env to link in libXm and libXt +RUN set -ex \ + && cd /opt/conda/lib \ + && unlink libuuid.so \ + && unlink libuuid.so.1 \ + && ln -s /lib64/libuuid.so.1.3.0 libuuid.so \ + && ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1 + +# install libgfortran.so.3 and create missing link +RUN set -ex \ + && yum install -y gcc-gfortran \ + && cd /lib64 \ + && ( test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so ) + +# copy repo +COPY . /opt/isce2/src/isce2 + +# build ISCE +RUN set -ex \ + && . /opt/conda/bin/activate root \ + && cd /opt/isce2/src/isce2 \ + && source docker/build_env.sh \ + && mkdir -p $BUILD_DIR \ + && cp docker/SConfigISCE.cuda configuration/SConfigISCE \ + && scons install \ + && cp docker/isce_env.sh $ISCE_INSTALL_ROOT \ + && cd /tmp \ + && rm -rf /opt/isce2/src \ + && mkdir -p /tmp/rpm-build/opt \ + && mv $ISCE_INSTALL_ROOT /tmp/rpm-build/opt \ + && curl -s https://api.github.com/repos/$ISCE_ORG/isce2/git/refs/heads/master \ + > /tmp/rpm-build/opt/isce2/version.json \ + && hash=$(cat /tmp/rpm-build/opt/isce2/version.json | jq -r .object.sha) \ + && short_hash=$(echo $hash | cut -c1-5) \ + && fpm -s dir -t rpm -C /tmp/rpm-build --name isce \ + --prefix=/ --version=2.3 --provides=isce \ + --maintainer=piyush.agram@jpl.nasa.gov \ + --description="InSAR Scientific Computing Environment v2 (${hash})" + +FROM hysds/cuda-pge-base:latest + +# Set an encoding to make things work smoothly. +ENV LANG en_US.UTF-8 + +# install ISCE from RPM +COPY --from=0 /tmp/isce-2.3-1.x86_64.rpm /tmp/isce-2.3-1.x86_64.rpm + +# install isce and its minimal requirements +RUN set -ex \ + && sudo /opt/conda/bin/conda install --yes \ + gdal \ + h5py \ + libgdal \ + pytest \ + numpy \ + fftw \ + scipy \ + hdf4 \ + hdf5 \ + netcdf4 \ + && sudo yum update -y \ + && sudo yum install -y uuid-devel x11-devel motif-devel gcc-gfortran \ + && cd /opt/conda/lib \ + && sudo unlink libuuid.so \ + && sudo unlink libuuid.so.1 \ + && sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so \ + && sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1 \ + && cd /lib64 \ + && ( test -f libgfortran.so || sudo ln -sv libgfortran.so.*.* libgfortran.so ) \ + && sudo yum install -y /tmp/isce-2.3-1.x86_64.rpm \ + && sudo yum clean all \ + && sudo rm -rf /var/cache/yum \ + && sudo rm /tmp/isce-2.3-1.x86_64.rpm diff --git a/docker/README.md b/docker/README.md index a6ae10d..4538b4e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -12,5 +12,9 @@ ``` 1. Build image: ``` - docker build --rm --force-rm -t hysds/isce_giant -f docker/Dockerfile . + docker build --rm --force-rm -t hysds/isce2:latest -f docker/Dockerfile . + ``` + For cuda version: + ``` + docker build --rm --force-rm -t hysds/isce2:latest-cuda -f docker/Dockerfile.cuda . ``` diff --git a/docker/SConfigISCE.cuda b/docker/SConfigISCE.cuda new file mode 100644 index 0000000..8db9971 --- /dev/null +++ b/docker/SConfigISCE.cuda @@ -0,0 +1,41 @@ +# The directory in which ISCE will be built +PRJ_SCONS_BUILD = $ISCE_BUILD_ROOT/isce2 + +# The directory into which ISCE will be installed +PRJ_SCONS_INSTALL = $ISCE_INSTALL_ROOT/isce + +# The location of libraries, such as libstdc++, libfftw3 (for most system +# it's /usr/lib and/or /usr/local/lib/ and/or /opt/local/lib) +LIBPATH = /usr/lib64 /usr/lib /opt/conda/lib + +# The location of Python.h. If you have multiple installations of python +# make sure that it points to the right one +CPPPATH = /opt/conda/include/python3.7m /opt/conda/lib/python3.7/site-packages/numpy/core/include /opt/conda/include /usr/include + +# The location of the fftw3.h (most likely something like /usr/include or +# /usr/local/include /opt/local/include +FORTRANPATH = /usr/include /opt/conda/include + +# The location of your Fortran compiler. If not specified it will use the system one +FORTRAN = /bin/gfortran + +# The location of your C compiler. If not specified it will use the system one +CC = /bin/gcc + +# The location of your C++ compiler. If not specified it will use the system one +CXX = /bin/g++ + +#libraries needed for mdx display utility +MOTIFLIBPATH = /usr/lib64 # path to libXm.dylib +X11LIBPATH = /usr/lib64 # path to libXt.dylib +MOTIFINCPATH = /usr/include # path to location of the Xm + # directory with various include files (.h) +X11INCPATH = /usr/include # path to location of the X11 directory + # with various include files + +# list of paths to search for shared libraries when running programs +RPATH = /opt/conda/lib /usr/lib64 /usr/lib + +#Explicitly enable cuda if needed +ENABLE_CUDA = True +#CUDA_TOOLKIT_PATH = $YOUR_CUDA_INSTALLATION #/usr/local/cuda diff --git a/docker/build_env.sh b/docker/build_env.sh index 0147310..79b2d15 100755 --- a/docker/build_env.sh +++ b/docker/build_env.sh @@ -1,5 +1,5 @@ -export BUILD_DIR=/root/build-isce -export ISCE_SRC=$HOME/isce2 +export ISCE_INSTALL_ROOT=/opt/isce2 +export ISCE_SRC=$ISCE_INSTALL_ROOT/src/isce2 +export BUILD_DIR=$ISCE_INSTALL_ROOT/build-isce export SCONS_CONFIG_DIR=$ISCE_SRC/configuration export ISCE_BUILD_ROOT=$BUILD_DIR/build -export ISCE_INSTALL_ROOT=/opt/isce2 diff --git a/scons_tools/cuda.py b/scons_tools/cuda.py index 9dce35b..4edaf12 100644 --- a/scons_tools/cuda.py +++ b/scons_tools/cuda.py @@ -52,7 +52,7 @@ def generate(env): # default flags for the NVCC compiler env['STATICNVCCFLAGS'] = '' env['SHAREDNVCCFLAGS'] = '' - env['ENABLESHAREDNVCCFLAG'] = '-shared -Xcompiler -fPIC' + env['ENABLESHAREDNVCCFLAG'] = '-std=c++11 -shared -Xcompiler -fPIC -I/opt/conda/include' # default NVCC commands env['STATICNVCCCMD'] = '$NVCC -o $TARGET -c $NVCCFLAGS $STATICNVCCFLAGS $SOURCES'