KJ135
/
ISCE_INSAR
mirror of http://172.16.0.12:5000/WBJY/ISCE_INSAR.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #442 from rtburns-jpl/ci-oracle-linux 

Switch CI base images to ubuntu 20.04
LT1AB
Ryan Burns 2022-02-24 10:46:03 -08:00 committed by GitHub
parent 2717095323 f96f9c998f
commit 566ebfb9a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 82 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
.circleci/config.yml
View File

@ -2,43 +2,29 @@ version: 2.1
jobs:
test-cmake:
docker:
- image: hysds/pge-base:latest
user: root
- image: ubuntu:20.04
steps:
- checkout:
path: /root/project/src
- run:
name: Install development tools
command: |
set -ex
pwd
yum update -y
yum groupinstall -y "development tools"
- run:
name: Install ISCE requirements
command: |
set -ex
pwd
mkdir config build install
. /opt/conda/bin/activate root
conda install -y cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 libgcc libstdcxx-ng cmake astropy pybind11
yum install -y x11-devel motif-devel jq gcc-gfortran opencv opencv-devel opencv-python
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y cmake cython3 git libfftw3-dev libgdal-dev libhdf4-alt-dev libhdf5-dev libopencv-dev python3-gdal python3-h5py python3-numpy python3-scipy
- run:
name: Build and Install ISCE
command: |
set -ex
cd /root/project/src
. /opt/conda/bin/activate root
mkdir build
cd build
INSTALLPATH=/opt/conda
MODPATH=$(python3 -c "import site; print(site.getsitepackages()[0])")
# convert to relative path
MODPATH=$(realpath --relative-to=$INSTALLPATH $MODPATH)
cmake .. -DCMAKE_INSTALL_PREFIX=$INSTALLPATH -DPYTHON_MODULE_DIR=$MODPATH
MODPATH=$(python3 -c "import site; print(site.getsitepackages()[-1])")
cmake .. -DCMAKE_INSTALL_PREFIX=install -DPYTHON_MODULE_DIR=$MODPATH
make install VERBOSE=y
- run:
@ -46,7 +32,6 @@ jobs:
command: |
set -ex
cd /root/project/src/build
. /opt/conda/bin/activate root
ctest --output-on-failure
ISCE2DIR=$(python3 -c "import os, isce2; print(os.path.dirname(isce2.__file__))" | tail -n 1)
export PATH=$ISCE2DIR/applications:$PATH
@ -60,37 +45,20 @@ jobs:
test:
docker:
- image: hysds/pge-base:latest
user: root
- image: ubuntu:20.04
steps:
- checkout:
path: /root/project/src
- run:
name: Install development tools
command: |
set -ex
pwd
yum update -y
yum groupinstall -y "development tools"
- run:
name: Install ISCE requirements
command: |
set -ex
pwd
mkdir config build install
. /opt/conda/bin/activate root
conda install --yes cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 libgcc libstdcxx-ng cmake astropy pybind11
yum install -y uuid-devel x11-devel motif-devel jq gcc-gfortran opencv opencv-devel opencv-python
ln -s /opt/conda/bin/cython /opt/conda/bin/cython3
cd /opt/conda/lib
unlink libuuid.so
unlink libuuid.so.1
ln -s /lib64/libuuid.so.1.3.0 libuuid.so
ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1
cd /lib64
test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y scons cython3 git libfftw3-dev libgdal-dev libhdf4-alt-dev libhdf5-dev libmotif-dev libopencv-dev libx11-dev python3-gdal python3-h5py python3-numpy python3-scipy
- run:
name: Build SConfigISCE and setup dirs
@ -100,10 +68,10 @@ jobs:
cd config
echo "PRJ_SCONS_BUILD = /root/project/build" > SConfigISCE
echo "PRJ_SCONS_INSTALL = /root/project/install/isce" >> SConfigISCE
echo "LIBPATH = /usr/lib64 /usr/lib /opt/conda/lib" >> SConfigISCE
python_inc="$(echo /opt/conda/include/python3.* /opt/conda/lib/python3.*/site-packages/numpy/core/include)"
echo "CPPPATH = $python_inc /opt/conda/include /usr/include" >> SConfigISCE
echo "FORTRANPATH = /usr/include /opt/conda/include" >> SConfigISCE
echo "LIBPATH = /usr/lib64 /usr/lib /usr/lib/x86_64-linux-gnu" >> SConfigISCE
python_inc="/usr/include/python3.8 /usr/lib/python3/dist-packages/numpy/core/include"
echo "CPPPATH = $python_inc /usr/include /usr/include/gdal /usr/include/opencv4" >> SConfigISCE
echo "FORTRANPATH = /usr/include" >> SConfigISCE
echo "FORTRAN = /bin/gfortran" >> SConfigISCE
echo "CC = /bin/gcc" >> SConfigISCE
echo "CXX = /bin/g++" >> SConfigISCE
@ -111,7 +79,7 @@ jobs:
echo "X11LIBPATH = /usr/lib64" >> SConfigISCE
echo "MOTIFINCPATH = /usr/include" >> SConfigISCE
echo "X11INCPATH = /usr/include" >> SConfigISCE
echo "RPATH = /opt/conda/lib /usr/lib64 /usr/lib" >> SConfigISCE
echo "RPATH = /usr/lib64 /usr/lib" >> SConfigISCE
cat SConfigISCE
- run:
@ -119,10 +87,7 @@ jobs:
command: |
set -ex
pwd
. /opt/conda/bin/activate root
cd src
export PATH="/opt/conda/bin:$PATH"
export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH"
SCONS_CONFIG_DIR=/root/project/config scons install --skipcheck
- run:
@ -130,11 +95,9 @@ jobs:
command: |
set -ex
pwd
. /opt/conda/bin/activate root
ISCE_HOME=/root/project/install/isce
export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:/opt/conda/bin:$PATH"
export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:$PATH"
export PYTHONPATH="/root/project/install:$PYTHONPATH"
export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH"
topsApp.py --help --steps
stripmapApp.py --help --steps
python3 -c "import isce"

17
CMakeLists.txt
View File

@ -41,12 +41,17 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
endif()
if(NOT DEFINED PYTHON_MODULE_DIR)
set(PYTHON_MODULE_DIR packages CACHE PATH
"Python module directory (relative to install prefix)")
set(PYTHON_MODULE_DIR packages CACHE PATH "Python module directory")
endif()
if(NOT DEFINED ISCE2_PKG)
set(ISCE2_PKG ${PYTHON_MODULE_DIR}/isce2 CACHE PATH
"ISCE 2 python package install dir (relative to install prefix)")
"ISCE 2 python package install dir")
endif()
if(IS_ABSOLUTE "${ISCE2_PKG}")
set(ISCE2_PKG_FULL "${ISCE2_PKG}")
else()
set(ISCE2_PKG_FULL "${CMAKE_INSTALL_PREFIX}/${ISCE2_PKG}")
endif()
include(isce2_buildflags)
@ -95,3 +100,9 @@ else()
endif()
install(CODE "execute_process(COMMAND
${CMAKE_COMMAND} -E create_symlink ${symsrc} ${symdest})")
# Enable native packaging using CPack
if(NOT CPACK_PACKAGE_CONTACT)
set(CPACK_PACKAGE_CONTACT "Ryan Burns <rtburns@jpl.nasa.gov>")
endif()
include(CPack)

7
applications/CMakeLists.txt
View File

@ -32,11 +32,12 @@ install(PROGRAMS ${files}
# Symlink apps into PREFIX/bin so they are on the $PATH
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory \
${CMAKE_INSTALL_PREFIX}/bin)"
${CMAKE_INSTALL_FULL_BINDIR})"
)
foreach(file ${files})
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \
${CMAKE_INSTALL_PREFIX}/${ISCE2_PKG}/applications/${file} \
${CMAKE_INSTALL_PREFIX}/bin/${file})"
${ISCE2_PKG_FULL}/applications/${file} \
${CMAKE_INSTALL_FULL_BINDIR}/${file})"
)
endforeach()

145
docker/Dockerfile
View File

@ -1,115 +1,66 @@
FROM hysds/dev:latest
FROM ubuntu:20.04 as builder
# Set an encoding to make things work smoothly.
ENV LANG en_US.UTF-8
ENV TZ US/Pacific
ARG DEBIAN_FRONTEND=noninteractive
# Set ISCE repo
ENV ISCE_ORG isce-framework
# set to root user
USER root
# install tools for RPM generation
RUN set -ex \
&& yum update -y \
&& yum groupinstall -y "development tools" \
&& yum install -y \
make ruby-devel rpm-build rubygems \
&& gem install ffi -v 1.12.2 \
&& gem install --no-ri --no-rdoc fpm -v 1.11.0
# install isce requirements
RUN set -ex \
&& . /opt/conda/bin/activate root \
&& conda install --yes \
cython \
gdal \
git \
h5py \
libgdal \
pytest \
numpy \
fftw \
scipy \
scons \
hdf4 \
hdf5 \
libgcc \
libstdcxx-ng \
cmake \
&& yum install -y uuid-devel x11-devel motif-devel jq \
opencv opencv-devel opencv-python \
&& ln -sf /opt/conda/bin/cython /opt/conda/bin/cython3 \
&& mkdir -p /opt/isce2/src
# override system libuuid into conda env to link in libXm and libXt
RUN set -ex \
&& cd /opt/conda/lib \
&& unlink libuuid.so \
&& unlink libuuid.so.1 \
&& ln -s /lib64/libuuid.so.1.3.0 libuuid.so \
&& ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1
# install libgfortran.so.3 and create missing link
RUN set -ex \
&& yum install -y gcc-gfortran \
&& cd /lib64 \
&& ( test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so )
&& apt-get update \
&& apt-get install -y \
cmake \
cython3 \
git \
libfftw3-dev \
libgdal-dev \
libhdf4-alt-dev \
libhdf5-dev \
libopencv-dev \
ninja-build \
python3-gdal \
python3-h5py \
python3-numpy \
python3-scipy \
&& echo done
# copy repo
COPY . /opt/isce2/src/isce2
# build ISCE
RUN set -ex \
&& . /opt/conda/bin/activate root \
&& cd /opt/isce2/src/isce2 \
&& source docker/build_env.sh \
&& mkdir -p $BUILD_DIR \
&& cp docker/SConfigISCE configuration/SConfigISCE \
&& scons install \
&& cp docker/isce_env.sh $ISCE_INSTALL_ROOT \
&& cd /tmp \
&& mkdir -p /tmp/rpm-build/opt \
&& mv $ISCE_INSTALL_ROOT /tmp/rpm-build/opt \
&& curl -s https://api.github.com/repos/$ISCE_ORG/isce2/git/refs/heads/main \
> /tmp/rpm-build/opt/isce2/version.json \
&& hash=$(cat /tmp/rpm-build/opt/isce2/version.json | jq -r .object.sha) \
&& short_hash=$(echo $hash | cut -c1-5) \
&& fpm -s dir -t rpm -C /tmp/rpm-build --name isce \
--prefix=/ --version=2.3 --provides=isce \
--maintainer=piyush.agram@jpl.nasa.gov \
--description="InSAR Scientific Computing Environment v2 (${hash})"
&& mkdir build && cd build \
&& cmake .. \
-DPYTHON_MODULE_DIR="$(python3 -c 'import site; print(site.getsitepackages()[-1])')" \
-DCMAKE_INSTALL_PREFIX=install \
&& make -j8 install \
&& cpack -G DEB \
&& cp isce*.deb /tmp/
FROM hysds/pge-base:latest
FROM ubuntu:20.04
# Set an encoding to make things work smoothly.
ENV LANG en_US.UTF-8
ENV TZ US/Pacific
ARG DEBIAN_FRONTEND=noninteractive
# install ISCE from RPM
COPY --from=0 /tmp/isce-2.3-1.x86_64.rpm /tmp/isce-2.3-1.x86_64.rpm
# install isce and its minimal requirements
RUN set -ex \
&& sudo /opt/conda/bin/conda install --yes \
gdal \
h5py \
libgdal \
pytest \
numpy \
fftw \
scipy \
hdf4 \
hdf5 \
&& sudo yum update -y \
&& sudo yum install -y uuid-devel x11-devel motif-devel gcc-gfortran \
&& cd /opt/conda/lib \
&& sudo unlink libuuid.so \
&& sudo unlink libuuid.so.1 \
&& sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so \
&& sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1 \
&& cd /lib64 \
&& ( test -f libgfortran.so || sudo ln -sv libgfortran.so.*.* libgfortran.so ) \
&& sudo yum install -y /tmp/isce-2.3-1.x86_64.rpm \
&& sudo yum clean all \
&& sudo rm -rf /var/cache/yum \
&& sudo rm /tmp/isce-2.3-1.x86_64.rpm
&& apt-get update \
&& apt-get install -y \
libfftw3-3 \
libgdal26 \
libhdf4-0 \
libhdf5-103 \
libopencv-core4.2 \
libopencv-highgui4.2 \
libopencv-imgproc4.2 \
python3-gdal \
python3-h5py \
python3-numpy \
python3-scipy \
&& echo done
# install ISCE from DEB
COPY --from=builder /tmp/isce*.deb /tmp/isce2.deb
RUN dpkg -i /tmp/isce2.deb