KJ135
/
ISCE_INSAR
mirror of http://172.16.0.12:5000/WBJY/ISCE_INSAR.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #442 from rtburns-jpl/ci-oracle-linux 

Switch CI base images to ubuntu 20.04
LT1AB
Ryan Burns 2022-02-24 10:46:03 -08:00 committed by GitHub
parent 2717095323 f96f9c998f
commit 566ebfb9a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 82 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
.circleci/config.yml
View File

@ -2,43 +2,29 @@ version: 2.1
jobs: jobs:
test-cmake: test-cmake:
docker: docker:
- image: hysds/pge-base:latest - image: ubuntu:20.04
user: root
steps: steps:
- checkout: - checkout:
path: /root/project/src path: /root/project/src
- run:
name: Install development tools
command: |
set -ex
pwd
yum update -y
yum groupinstall -y "development tools"
- run: - run:
name: Install ISCE requirements name: Install ISCE requirements
command: | command: |
set -ex set -ex
pwd pwd
mkdir config build install export DEBIAN_FRONTEND=noninteractive
. /opt/conda/bin/activate root apt-get update
conda install -y cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 libgcc libstdcxx-ng cmake astropy pybind11 apt-get install -y cmake cython3 git libfftw3-dev libgdal-dev libhdf4-alt-dev libhdf5-dev libopencv-dev python3-gdal python3-h5py python3-numpy python3-scipy
yum install -y x11-devel motif-devel jq gcc-gfortran opencv opencv-devel opencv-python
- run: - run:
name: Build and Install ISCE name: Build and Install ISCE
command: | command: |
set -ex set -ex
cd /root/project/src cd /root/project/src
. /opt/conda/bin/activate root
mkdir build mkdir build
cd build cd build
INSTALLPATH=/opt/conda MODPATH=$(python3 -c "import site; print(site.getsitepackages()[-1])")
MODPATH=$(python3 -c "import site; print(site.getsitepackages()[0])") cmake .. -DCMAKE_INSTALL_PREFIX=install -DPYTHON_MODULE_DIR=$MODPATH
# convert to relative path
MODPATH=$(realpath --relative-to=$INSTALLPATH $MODPATH)
cmake .. -DCMAKE_INSTALL_PREFIX=$INSTALLPATH -DPYTHON_MODULE_DIR=$MODPATH
make install VERBOSE=y make install VERBOSE=y
- run: - run:
@ -46,7 +32,6 @@ jobs:
command: | command: |
set -ex set -ex
cd /root/project/src/build cd /root/project/src/build
. /opt/conda/bin/activate root
ctest --output-on-failure ctest --output-on-failure
ISCE2DIR=$(python3 -c "import os, isce2; print(os.path.dirname(isce2.__file__))" | tail -n 1) ISCE2DIR=$(python3 -c "import os, isce2; print(os.path.dirname(isce2.__file__))" | tail -n 1)
export PATH=$ISCE2DIR/applications:$PATH export PATH=$ISCE2DIR/applications:$PATH
@ -60,37 +45,20 @@ jobs:
test: test:
docker: docker:
- image: hysds/pge-base:latest - image: ubuntu:20.04
user: root
steps: steps:
- checkout: - checkout:
path: /root/project/src path: /root/project/src
- run:
name: Install development tools
command: |
set -ex
pwd
yum update -y
yum groupinstall -y "development tools"
- run: - run:
name: Install ISCE requirements name: Install ISCE requirements
command: | command: |
set -ex set -ex
pwd pwd
mkdir config build install mkdir config build install
. /opt/conda/bin/activate root export DEBIAN_FRONTEND=noninteractive
conda install --yes cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 libgcc libstdcxx-ng cmake astropy pybind11 apt-get update
yum install -y uuid-devel x11-devel motif-devel jq gcc-gfortran opencv opencv-devel opencv-python apt-get install -y scons cython3 git libfftw3-dev libgdal-dev libhdf4-alt-dev libhdf5-dev libmotif-dev libopencv-dev libx11-dev python3-gdal python3-h5py python3-numpy python3-scipy
ln -s /opt/conda/bin/cython /opt/conda/bin/cython3
cd /opt/conda/lib
unlink libuuid.so
unlink libuuid.so.1
ln -s /lib64/libuuid.so.1.3.0 libuuid.so
ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1
cd /lib64
test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so
- run: - run:
name: Build SConfigISCE and setup dirs name: Build SConfigISCE and setup dirs
@ -100,10 +68,10 @@ jobs:
cd config cd config
echo "PRJ_SCONS_BUILD = /root/project/build" > SConfigISCE echo "PRJ_SCONS_BUILD = /root/project/build" > SConfigISCE
echo "PRJ_SCONS_INSTALL = /root/project/install/isce" >> SConfigISCE echo "PRJ_SCONS_INSTALL = /root/project/install/isce" >> SConfigISCE
echo "LIBPATH = /usr/lib64 /usr/lib /opt/conda/lib" >> SConfigISCE echo "LIBPATH = /usr/lib64 /usr/lib /usr/lib/x86_64-linux-gnu" >> SConfigISCE
python_inc="$(echo /opt/conda/include/python3.* /opt/conda/lib/python3.*/site-packages/numpy/core/include)" python_inc="/usr/include/python3.8 /usr/lib/python3/dist-packages/numpy/core/include"
echo "CPPPATH = $python_inc /opt/conda/include /usr/include" >> SConfigISCE echo "CPPPATH = $python_inc /usr/include /usr/include/gdal /usr/include/opencv4" >> SConfigISCE
echo "FORTRANPATH = /usr/include /opt/conda/include" >> SConfigISCE echo "FORTRANPATH = /usr/include" >> SConfigISCE
echo "FORTRAN = /bin/gfortran" >> SConfigISCE echo "FORTRAN = /bin/gfortran" >> SConfigISCE
echo "CC = /bin/gcc" >> SConfigISCE echo "CC = /bin/gcc" >> SConfigISCE
echo "CXX = /bin/g++" >> SConfigISCE echo "CXX = /bin/g++" >> SConfigISCE
@ -111,7 +79,7 @@ jobs:
echo "X11LIBPATH = /usr/lib64" >> SConfigISCE echo "X11LIBPATH = /usr/lib64" >> SConfigISCE
echo "MOTIFINCPATH = /usr/include" >> SConfigISCE echo "MOTIFINCPATH = /usr/include" >> SConfigISCE
echo "X11INCPATH = /usr/include" >> SConfigISCE echo "X11INCPATH = /usr/include" >> SConfigISCE
echo "RPATH = /opt/conda/lib /usr/lib64 /usr/lib" >> SConfigISCE echo "RPATH = /usr/lib64 /usr/lib" >> SConfigISCE
cat SConfigISCE cat SConfigISCE
- run: - run:
@ -119,10 +87,7 @@ jobs:
command: | command: |
set -ex set -ex
pwd pwd
. /opt/conda/bin/activate root
cd src cd src
export PATH="/opt/conda/bin:$PATH"
export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH"
SCONS_CONFIG_DIR=/root/project/config scons install --skipcheck SCONS_CONFIG_DIR=/root/project/config scons install --skipcheck
- run: - run:
@ -130,11 +95,9 @@ jobs:
command: | command: |
set -ex set -ex
pwd pwd
. /opt/conda/bin/activate root
ISCE_HOME=/root/project/install/isce ISCE_HOME=/root/project/install/isce
export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:/opt/conda/bin:$PATH" export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:$PATH"
export PYTHONPATH="/root/project/install:$PYTHONPATH" export PYTHONPATH="/root/project/install:$PYTHONPATH"
export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH"
topsApp.py --help --steps topsApp.py --help --steps
stripmapApp.py --help --steps stripmapApp.py --help --steps
python3 -c "import isce" python3 -c "import isce"

17
CMakeLists.txt
View File

@ -41,12 +41,17 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
endif() endif()
if(NOT DEFINED PYTHON_MODULE_DIR) if(NOT DEFINED PYTHON_MODULE_DIR)
set(PYTHON_MODULE_DIR packages CACHE PATH set(PYTHON_MODULE_DIR packages CACHE PATH "Python module directory")
"Python module directory (relative to install prefix)")
endif() endif()
if(NOT DEFINED ISCE2_PKG) if(NOT DEFINED ISCE2_PKG)
set(ISCE2_PKG ${PYTHON_MODULE_DIR}/isce2 CACHE PATH set(ISCE2_PKG ${PYTHON_MODULE_DIR}/isce2 CACHE PATH
"ISCE 2 python package install dir (relative to install prefix)") "ISCE 2 python package install dir")
endif()
if(IS_ABSOLUTE "${ISCE2_PKG}")
set(ISCE2_PKG_FULL "${ISCE2_PKG}")
else()
set(ISCE2_PKG_FULL "${CMAKE_INSTALL_PREFIX}/${ISCE2_PKG}")
endif() endif()
include(isce2_buildflags) include(isce2_buildflags)
@ -95,3 +100,9 @@ else()
endif() endif()
install(CODE "execute_process(COMMAND install(CODE "execute_process(COMMAND
${CMAKE_COMMAND} -E create_symlink ${symsrc} ${symdest})") ${CMAKE_COMMAND} -E create_symlink ${symsrc} ${symdest})")
# Enable native packaging using CPack
if(NOT CPACK_PACKAGE_CONTACT)
set(CPACK_PACKAGE_CONTACT "Ryan Burns <rtburns@jpl.nasa.gov>")
endif()
include(CPack)

7
applications/CMakeLists.txt
View File

@ -32,11 +32,12 @@ install(PROGRAMS ${files}
# Symlink apps into PREFIX/bin so they are on the $PATH # Symlink apps into PREFIX/bin so they are on the $PATH
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory \ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory \
${CMAKE_INSTALL_PREFIX}/bin)" ${CMAKE_INSTALL_FULL_BINDIR})"
) )
foreach(file ${files}) foreach(file ${files})
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \
${CMAKE_INSTALL_PREFIX}/${ISCE2_PKG}/applications/${file} \ ${ISCE2_PKG_FULL}/applications/${file} \
${CMAKE_INSTALL_PREFIX}/bin/${file})" ${CMAKE_INSTALL_FULL_BINDIR}/${file})"
) )
endforeach() endforeach()

145
docker/Dockerfile
View File

@ -1,115 +1,66 @@
FROM hysds/dev:latest FROM ubuntu:20.04 as builder
# Set an encoding to make things work smoothly. # Set an encoding to make things work smoothly.
ENV LANG en_US.UTF-8 ENV LANG en_US.UTF-8
ENV TZ US/Pacific
ARG DEBIAN_FRONTEND=noninteractive
# Set ISCE repo
ENV ISCE_ORG isce-framework
# set to root user
USER root
# install tools for RPM generation
RUN set -ex \ RUN set -ex \
&& yum update -y \ && apt-get update \
&& yum groupinstall -y "development tools" \ && apt-get install -y \
&& yum install -y \ cmake \
make ruby-devel rpm-build rubygems \ cython3 \
&& gem install ffi -v 1.12.2 \ git \
&& gem install --no-ri --no-rdoc fpm -v 1.11.0 libfftw3-dev \
libgdal-dev \
# install isce requirements libhdf4-alt-dev \
RUN set -ex \ libhdf5-dev \
&& . /opt/conda/bin/activate root \ libopencv-dev \
&& conda install --yes \ ninja-build \
cython \ python3-gdal \
gdal \ python3-h5py \
git \ python3-numpy \
h5py \ python3-scipy \
libgdal \ && echo done
pytest \
numpy \
fftw \
scipy \
scons \
hdf4 \
hdf5 \
libgcc \
libstdcxx-ng \
cmake \
&& yum install -y uuid-devel x11-devel motif-devel jq \
opencv opencv-devel opencv-python \
&& ln -sf /opt/conda/bin/cython /opt/conda/bin/cython3 \
&& mkdir -p /opt/isce2/src
# override system libuuid into conda env to link in libXm and libXt
RUN set -ex \
&& cd /opt/conda/lib \
&& unlink libuuid.so \
&& unlink libuuid.so.1 \
&& ln -s /lib64/libuuid.so.1.3.0 libuuid.so \
&& ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1
# install libgfortran.so.3 and create missing link
RUN set -ex \
&& yum install -y gcc-gfortran \
&& cd /lib64 \
&& ( test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so )
# copy repo # copy repo
COPY . /opt/isce2/src/isce2 COPY . /opt/isce2/src/isce2
# build ISCE # build ISCE
RUN set -ex \ RUN set -ex \
&& . /opt/conda/bin/activate root \
&& cd /opt/isce2/src/isce2 \ && cd /opt/isce2/src/isce2 \
&& source docker/build_env.sh \ && mkdir build && cd build \
&& mkdir -p $BUILD_DIR \ && cmake .. \
&& cp docker/SConfigISCE configuration/SConfigISCE \ -DPYTHON_MODULE_DIR="$(python3 -c 'import site; print(site.getsitepackages()[-1])')" \
&& scons install \ -DCMAKE_INSTALL_PREFIX=install \
&& cp docker/isce_env.sh $ISCE_INSTALL_ROOT \ && make -j8 install \
&& cd /tmp \ && cpack -G DEB \
&& mkdir -p /tmp/rpm-build/opt \ && cp isce*.deb /tmp/
&& mv $ISCE_INSTALL_ROOT /tmp/rpm-build/opt \
&& curl -s https://api.github.com/repos/$ISCE_ORG/isce2/git/refs/heads/main \
> /tmp/rpm-build/opt/isce2/version.json \
&& hash=$(cat /tmp/rpm-build/opt/isce2/version.json | jq -r .object.sha) \
&& short_hash=$(echo $hash | cut -c1-5) \
&& fpm -s dir -t rpm -C /tmp/rpm-build --name isce \
--prefix=/ --version=2.3 --provides=isce \
--maintainer=piyush.agram@jpl.nasa.gov \
--description="InSAR Scientific Computing Environment v2 (${hash})"
FROM hysds/pge-base:latest FROM ubuntu:20.04
# Set an encoding to make things work smoothly. # Set an encoding to make things work smoothly.
ENV LANG en_US.UTF-8 ENV LANG en_US.UTF-8
ENV TZ US/Pacific
ARG DEBIAN_FRONTEND=noninteractive
# install ISCE from RPM
COPY --from=0 /tmp/isce-2.3-1.x86_64.rpm /tmp/isce-2.3-1.x86_64.rpm
# install isce and its minimal requirements
RUN set -ex \ RUN set -ex \
&& sudo /opt/conda/bin/conda install --yes \ && apt-get update \
gdal \ && apt-get install -y \
h5py \ libfftw3-3 \
libgdal \ libgdal26 \
pytest \ libhdf4-0 \
numpy \ libhdf5-103 \
fftw \ libopencv-core4.2 \
scipy \ libopencv-highgui4.2 \
hdf4 \ libopencv-imgproc4.2 \
hdf5 \ python3-gdal \
&& sudo yum update -y \ python3-h5py \
&& sudo yum install -y uuid-devel x11-devel motif-devel gcc-gfortran \ python3-numpy \
&& cd /opt/conda/lib \ python3-scipy \
&& sudo unlink libuuid.so \ && echo done
&& sudo unlink libuuid.so.1 \
&& sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so \ # install ISCE from DEB
&& sudo ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1 \ COPY --from=builder /tmp/isce*.deb /tmp/isce2.deb
&& cd /lib64 \
&& ( test -f libgfortran.so || sudo ln -sv libgfortran.so.*.* libgfortran.so ) \ RUN dpkg -i /tmp/isce2.deb
&& sudo yum install -y /tmp/isce-2.3-1.x86_64.rpm \
&& sudo yum clean all \
&& sudo rm -rf /var/cache/yum \
&& sudo rm /tmp/isce-2.3-1.x86_64.rpm