Decide on FMS (mom5 branch) compile flags

[harshula]

The existing CMakeLists.txt contains the following compile flags:

FMS/CMakeLists.txt

Lines 40 to 64 in 38c7dbe

	if(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel")
	set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fno-alias -stack-temps -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -sox -traceback")
	set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -O2 -fp-model source")
	set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "${CMAKE_Fortran_FLAGS_RELWITHDEBINFO} -g")
	set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv")
	elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
	set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -no-pie -fcray-pointer -fdefault-real-8 -ffree-line-length-none -fno-range-check")
	set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -O2")
	set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "${CMAKE_Fortran_FLAGS_RELWITHDEBINFO} -g")
	set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -g -Wuninitialized -fcheck=bounds -Werror -ffpe-trap=invalid,zero,overflow")
	else ()
	message ("Unknown FORTRAN compiler default flags only...")
	endif()
	if(CMAKE_C_COMPILER_ID STREQUAL "Intel")
	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -sox -traceback")
	set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -debug")
	set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2 -debug minimal")
	elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU")
	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -frecord-gcc-switches")
	set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -Werror -Wuninitialized -Wno-stringop-overflow")
	set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2")
	else ()
	message ("Unknown C compiler default flags only...")
	endif()

The upstream FMS compile flags are: 02bdd2f

The MOM5 compile flags are: https://github.com/ACCESS-NRI/MOM5/blob/6d8768648ffa2326b4817c3de9ccb25e9d808585/bin/mkmf.template.nci#L39-L51

Which of them should we use?

[harshula]

Example of compilation flags from MOM5 with git subtree FMS (spack install mom5@master %intel@2021.2.0 target=x86_64): mpifort -Duse_netCDF -Duse_libMPI -DACCESS_OM -DUSE_OCEAN_BGC -fpp -Wp,-w -I/opt/release/linux-rocky8-x86_64/intel-2021.2.0/oasis3-mct-master-df2rx2hjfd2xmwfu34nzfkdugo34voew/include/psmile.MPI1 -I/opt/release/linux-rocky8-x86_64/intel-2021.2.0/oasis3-mct-master-df2rx2hjfd2xmwfu34nzfkdugo34voew/include -I/opt/release/linux-rocky8-x86_64/intel-2021.2.0/libaccessom2-master-ss24s7pyzzhbzck5dsqjd2z4ca3lavkw/include -I/opt/release/linux-rocky8-x86_64/intel-2021.2.0/netcdf-fortran-4.6.1-dutskkuweiulkdk3vmdzvmshgc4wtuo2/include -fno-alias -safe-cray-ptr -fpe0 -ftz -assume byterecl -i4 -r8 -nowarn -check noarg_temp_created -assume nobuffered_io -convert big_endian -grecord-gcc-switches -align all -fp-model precise -fp-model source -align all -g3 -O2 -xCORE-AVX2 -debug all -check none -traceback -c -I/tmp/root/spack-stage/spack-stage-mom5-master-vgak75luzyoz6ietfh46jpj2laox6xxf/spack-src/src/shared/include /tmp/root/spack-stage/spack-stage-mom5-master-vgak75luzyoz6ietfh46jpj2laox6xxf/spack-src/src/shared/fms/fms.F90

[harshula]

Looks like src/shared/fms/fms.F90 is currently compiled with MOM5 compile flags. So, if we want to be consistent with existing MOM5 compilations, I guess we have to build FMS with MOM5 compile flags?

[dougiesquire]

I guess we have to build FMS with MOM5 compile flags?

Agreed - I think we want to consistent with what was being done before

[harshula]

Hi @manodeep ,

What compiler flags should we use for IntelLLVM? We could start with with FMS upstream's compiler flags:

cmake/compiler_flags_IntelLLVM_C.cmake:

set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -traceback")
set( CMAKE_C_FLAGS_RELEASE "-qno-opt-dynamic-align -O2 -debug minimal")
set( CMAKE_C_FLAGS_DEBUG "-O0 -g -ftrapuv")
set( CMAKE_C_LINK_FLAGS "")

cmake/compiler_flags_IntelLLVM_Fortran.cmake:

set(r4_flags "-real-size 32") # Fortran flags for 32BIT precision
set(r8_flags "-real-size 64") # Fortran flags for 64BIT precision
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -align array64byte -nowarn -traceback")
set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal -nowarn -qoverride-limits -qno-opt-dynamic-align")
set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv")
set(CMAKE_Fortran_LINK_FLAGS "")

[manodeep]

@harshula Those flags look fine to me. Do you know why the oneAPI compiler returns a fatal error?

[harshula]

Hi @manodeep

Have a look at https://github.com/ACCESS-NRI/FMS/blob/development/CMakeLists.txt and see what happens when CMAKE_Fortran_COMPILER_ID is IntelLLVM.

I should be able to get back to FMS this week and add IntelLLVM support.

Can you please help me by checking each of the compiler flags in #1 (comment) and tell me if any are incompatible with IFX? If they are, please suggest an alternative.

[manodeep]

Hi @manodeep

Have a look at https://github.com/ACCESS-NRI/FMS/blob/development/CMakeLists.txt and see what happens when CMAKE_Fortran_COMPILER_ID is IntelLLVM.

@harshula Uhh cmake returns a fatal error here - which is what I was sayinghere. It would have made sense to treat the oneAPI as the same as classic - do you know why the choice was made to return a "fatal error"?

I should be able to get back to FMS this week and add IntelLLVM support.

Fantastic! If it would be helpful, I am happy to jump on a zoom call and try to compile those oneAPI packages out together.

Can you please help me by checking each of the compiler flags in #1 (comment) and tell me if any are incompatible with IFX? If they are, please suggest an alternative.

Those flags are compatible with ifx. -ip is a classic Intel flag that I know is not compatible with ifx. Another flag that I suspect is no longer supported is -mp1 - which is used in UM7; possibly elsewhere.

I would really like to add -flto to the compiler flags - the reports are that the Intel optimisations are only enabled when -flto is set; otherwise, you get the default LLVM optimiser. See this comment