buildSettings.mk

# This file defines standard commands and compiler flags. It is included
# before vars.mk, which uses the variables defined here to construct the
# final compiler flag variables that are used in C/C++ compilation.
#
# Final values for CFLAGS, CPPFLAGS, CXXFLAGS, LDFLAGS, NVCCFLAGS, and
# NVCCLDFLAGS (including include/library search paths) are constructed in
# vars.mk,


# get system type (Darwin, Linux)
SYSTYPE ?= $(shell uname -s)


#####################################################################
#
# Standard commands
#
#####################################################################
SHELL   ?= /bin/sh
RM      ?= /bin/rm
CP      ?= /bin/cp
MV      ?= /bin/mv
CHMOD   ?= /bin/chmod
MKDIR   ?= /bin/mkdir
INSTALL ?= install -p
TCLSH   ?= ${FSLDIR}/bin/fsltclsh

#####################################################################
#
# Architecture-independent compiler/linker flags
#
#####################################################################

# Platform independent compiler/linker options may be added to
# the following ARCH* variables. These variables are *always*
# added to compiler calls, for all platforms.
#
# The ARCH* variables here are added to the end of the final
# FLAGS variables, which means that any options specified here
# will typically take precedence over options provided by the
# environment.

# Compiler flags for C projects.
#  - C99 as minimum source compatibility standard
#  - Position independent code essential for
#    compiling/using shared libraries
ARCHCFLAGS = -std=c99 -fPIC

# Compiler flags for C++ projects.
#  - C++17 as minimum source compatibility standard
#  - Position independent code essential for
#    compiling/using shared libraries
ARCHCXXFLAGS = -std=c++17 -fPIC -g -O0

# Preprocesor flags for C/C++ projects.
ARCHCPPFLAGS = -g -O0

# Linker flags for all projects
ARCHLDFLAGS = 

# Libraries available for linking by all projects.
ARCHLIBS = 

# Compiler flags for CUDA projects.
#  - Define ARMA_ALLOW_FAKE_GCC, otherwise nvcc-compiled
#    armadillo structures may have different byte-alignment
#    to equivalent g++-compiled structures (see
#    include/armadillo_bits/compiler_setup.hpp)
#
#  - Set -std=c++14. This is the newest C++ standard
#    supported by nvcc for CUDA<11.0.
# ARCHNVCCFLAGS = -std=c++14 -fPIC
ARCHNVCCFLAGS = -DARMA_ALLOW_FAKE_GCC -std=c++17 -fPIC -g -O0

# Linker flags for CUDA projects.
ARCHNVCCLDFLAGS = -std=c++17 -fPIC  

#####################################################################
#
# Default compiler commands, flags and basic external libraries for
# macOS.
#
#####################################################################

# Drop -rpath flags from LDFLAGS if it is set in
# the environment - we explicitly add them below
# such that executables will be able to find libs
# in $(pwd), $FSLDEVDIR/lib, and $FSLDIR/lib *in
# that order*. This is so that executables
# located in the project source dir can be
# executed in place with sensible behaviour (i.e.
# any shared libs located in the same dir will
# take precedence over libs of the same name in
# $FSLDIR/lib/).
#
# Fortunately -Wl,-rpath is used for both g++ and
# clang, so we can cover both with one substitution.
COMMA    := ,
_LDFLAGS := $(patsubst -Wl${COMMA}-rpath%,,${LDFLAGS})
LDFLAGS   = ${_LDFLAGS}

# Also drop -L options from LDFLAGS if set in the
# environment, as we want to set our own
# precedence rules as above ($FSLDEVDIR > $FSLDIR)
# - this is done in vars.mk
_LDFLAGS := $(patsubst -L%,,${_LDFLAGS})
LDFLAGS   = ${_LDFLAGS}

# Construct all rpath options, and add them to
# LDFLAGS.  We keep them in a separate variable
# so they can be added to NVCCLDFLAGS as well.
RPATHS    =

ifeq ($(SYSTYPE), Darwin)

  # CC, CXX, CFLAGS, CXXFLAGS, and LDFLAGS
  # may be overridden by the environment.
  CC       ?= clang
  CXX      ?= clang++
  CFLAGS   ?= -arch x86_64 -Wall -pedantic
  CXXFLAGS ?= -arch x86_64 -Wall -pedantic
  RPATHS   += -Wl,-rpath,"@executable_path/"
  # Only add FSLDEVDIR if different to FSLDIR
  ifneq (${FSLDEVDIR}, ${FSLDIR})
    # RPATHS += -Wl,-rpath,"${FSLDEVDIR}/lib"
  endif
  # RPATHS   += -Wl,-rpath,"${FSLDIR}/lib"
  ARCHLIBS +=  -llapack -lblas -lz -lm
  # -L /usr/local/lib/python3.10/dist-packages/ray/core  -ljemalloc 

  # Old versions of libxml++ (which is used by
  # ciftilib) use std::auto_ptr, which has been
  # removed from the llvm C++17 implementation.
  # Adding this flag is a hack which allows us
  # to continue to use std::auto_ptr.
  ARCHCPPFLAGS += -D_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR

  # Some standard C++ features may be hidden when
  # being compiled on a macOS version which does
  # not support those features. But we use the
  # conda provided libc++.
  # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-skd
  ARCHCPPFLAGS += -D_LIBCPP_DISABLE_AVAILABILITY

  # On macOS, we need to change the install
  # name of shared libs so that they can be
  # looked up according to the rpath of
  # executables that depend on them. Note
  # that we give shared libs a suffix of
  # .so (not .dylib) on both linux and
  # macOS.
  %.so : RPATHS += -Wl,-install_name,@rpath/$@

  ARCHLDFLAGS += ${RPATHS}
endif

#####################################################################
#
# Default compiler commands, flags and basic external libraries for
# Linux.
#
#####################################################################
ifeq ($(SYSTYPE), Linux)

  # CC, CXX, CFLAGS, CXXFLAGS, and LDFLAGS
  # may be overridden by the environment.
  CC       ?= gcc
  CXX      ?= g++
  CFLAGS   ?= -fexpensive-optimizations \
              -Wall -pedantic -Wno-long-long
  CXXFLAGS ?= -fexpensive-optimizations \
              -Wall -pedantic -Wno-long-long
  # RPATHS   += -Wl,-rpath,'$$ORIGIN' \
  #             -Wl,-rpath-link,'$$ORIGIN'
  
  # Only add FSLDEVDIR if different to FSLDIR
  ifneq (${FSLDEVDIR}, ${FSLDIR})
    # RPATHS  += -Wl,-rpath,'${FSLDEVDIR}/lib' \
    #            -Wl,-rpath-link,'${FSLDEVDIR}/lib'
  endif
  # RPATHS    += -Wl,-rpath,'${FSLDIR}/lib' \
  #              -Wl,-rpath-link,'${FSLDIR}/lib'
  LDFLAGS   += ${RPATHS}

  # Libraries which are implicitly available
  # to all projects for linking
  ARCHCXXFLAGS += -pthread
  ARCHLIBS     += -llapack -lblas -lz -lm
  # -L /usr/local/lib/python3.10/dist-packages/ray/core  -ljemalloc 
  # -L /usr/local/lib/python3.10/dist-packages/ray/core  -ljemalloc 
endif

#####################################################################
#
# Default compiler commands, flags, and basic external libraries for
# CUDA projects. nvcc must be on the $PATH, or $NVCC must be set.
#
#####################################################################
NVCC ?= $(shell which nvcc 2> /dev/null)
ifneq (${NVCC}, )

  # NVCC, GENCODEFLAGS, NVCCFLAGS, NVCCLDFLAGS,
  # and CUDA_DYNAMIC may be specified/overridden
  # by the environment.
  CUDA_HOME     = $(realpath $(dir ${NVCC})/../)
  CUDA_VER      = $(shell ${NVCC} --version | grep release | cut -d ' ' -f 5 | tr -d ',')
  GENCODEFLAGS ?= $(shell ${FSLCONFDIR}/supportedGencodes.sh ${CUDA_VER})

  # We use nvcc to compile .cu files, g++ to
  # compile .cc/.cpp files, and nvcc to perform
  # the final linking. So we need to manage
  # compiler/linker flags carefully.

  # NVCCFLAGS is used for compiling .cu files
  # (and final linking) - it contains nvcc-
  # specific options.
  #
  # See vars.mk for the final construction of
  # these variables

  # We specify the compiler to use with -ccbin, as
  # nvcc might otherwise naively call "g++", rather
  # than honouring $(CXX). We specifically use
  # "-ccbin" rather than "--compiler-bindir" here,
  # because the conda-forge nvcc wrapper checks for
  # -ccbin, and adds its own if not present.
  NVCCFLAGS    ?= ${GENCODEFLAGS} \
                  -ccbin $(CXX)
  NVCCLDFLAGS  ?= -L${CUDA_HOME}/lib         \
                  -L${CUDA_HOME}/lib64       \
                  -L${CUDA_HOME}/lib/stubs   \
                  -L${CUDA_HOME}/lib64/stubs \
                  -L/usr/local/cuda/lib      \
                  $(subst -Wl${COMMA},--linker-options ,${RPATHS})

  # Link CUDA libraries dynamically, if compilation
  # was invoked with "make CUDA_DYNAMIC=1". The
  # default behaviour is to link statically.
  # Compiler / linker options vary depending on the
  # CUDA version
  _CUDALIBS = -lcuda
  ifndef CUDA_DYNAMIC
    # The cuda/cudart/cudadevrt libs are handled by nvcc.
    # Other components of the CUDA toolkit are provided
    # as both dynamic and static libraries. "_CUDALIBS"
    # is what is used to construct the final NVCCLDFLAGS
    # (see vars.mk).
    #
    # --cudadevrt was added in CUDA/nvcc 10.*
    #
    # In CUDA 10.* and newer, cublas_static requires
    # cublas_Lt_static, in addition to culibos
    ifeq ($(patsubst 9.%,,${CUDA_VER}),)
      # CUBLAS_STATIC = -lcublas -lculibos
      CUBLAS_STATIC = -lcublas -lcublasLt -l cudart -l cudadevrt 

      # -L /usr/local/lib/python3.10/dist-packages/ray/core  -ljemalloc
      # NVCCLDFLAGS  += --cudart=static
    else
      # CUBLAS_STATIC = -lcublas -lcublasLt -lculibos
      CUBLAS_STATIC = -lcublas -lcublasLt -l cudart -l cudadevrt 
      # -L /usr/local/lib/python3.10/dist-packages/ray/core  -ljemalloc
      # NVCCLDFLAGS  += --cudart=static --cudadevrt=static
	endif

    # Other CUDA toolkit components will
    # be added here on an as-needed basis.
    _CUDALIBS += $(subst -lcublas,${CUBLAS_STATIC}, \
                 $(subst -lcurand,-lcurand_static,${CUDALIBS}))
  else
    ifeq ($(patsubst 9.%,,${CUDA_VER}),)
      NVCCLDFLAGS += --cudart=shared
    else
      NVCCLDFLAGS += --cudart=shared --cudadevrt=static
    endif
    _CUDALIBS += ${CUDALIBS}
  endif
endif