first commit

head/config_parameters.h

+#ifndef __READ_PARAMETERS_H
+#define __READ_PARAMETERS_H
+
+#include "stdio.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+// config macro
+#define DEBUG_MODE
+
+typedef double REAL;
+// boundary lap size
+#define LAP 4
+// unknown number
+#define NVARS 5                         
+
+#define hipWarpSize 64
+
+typedef struct scheme_choose{
+    char *invis;
+    char *vis;
+}SCHEME_CHOOSE;
+
+#define Negatie_T_limit 100
+#define NegT_Max 500
+
+typedef struct TYPE_NegT_{
+    int Num_NegT;
+    int NT_Point[NegT_Max][3];
+}TYPE_NegT;
+
+typedef struct configItem_
+{
+    char name[1000];
+    char value[1000];
+} configItem;
+
+
+int ExtarctItem(char *src, char *name, char *value);
+int ItemNUM(FILE *file, char *Item_name, int *NameNUM);
+int PartItem(char *src, char part[][1000]);
+void ModifyItem(char *name, char *buff);
+void RemovalNUM(char *buff);
+void SearchItem(FILE *file, configItem *List, int configNum);
+void Schemes_Choose_ID(SCHEME_CHOOSE *scheme);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+

head/cuda_commen.h

+#ifndef _CUDA_HEAD_H
+#define _CUDA_HEAD_H
+
+#include "config_parameters.h"
+#include "cuda.h"
+#include "cuda_runtime.h"
+#include "mpi.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+
+#ifdef DEBUG_MODE
+#include "stdio.h"
+#define CUDA_CALL(CALL) \
+    {\
+        cudaError_t error;\
+        error = CALL;\
+        if(error != cudaSuccess){\
+            printf("In file \"%s\" , line %d ( \"%s\" ) , cuda call failed\n",__FILE__ , __LINE__ , __FUNCTION__);\
+            printf("Error code : %s  (%s)\n",cudaGetErrorString(error) , cudaGetErrorName(error));\
+            MPI_Abort(((MPI_Comm)0x44000000) , 1);\
+        }\
+    }
+#else
+#define CUDA_CALL(CALL) CALL;
+#endif
+
+#ifdef DEBUG_MODE
+#include "stdio.h"
+#define CUDA_LAUNCH(call)\
+    {\
+        cudaError_t error;\
+        call;\
+        error = cudaGetLastError();\
+        if( error != cudaSuccess){\
+            printf("In file \"%s\" , line %d ( \"%s\" ) , cuda call failed\n",__FILE__ , __LINE__ , __FUNCTION__);\
+            printf("Error code : %s  (%s)\n",cudaGetErrorString(error) , cudaGetErrorName(error));\
+            MPI_Abort(((MPI_Comm)0x44000000) , 1);\
+        }\
+    }
+#else
+#define CUDA_LAUNCH(call)\
+    call;
+#endif
+
+typedef struct cudaField_ {
+    REAL * ptr;
+    unsigned int pitch;
+    #ifdef __HIPCC__
+    unsigned int dummy0;
+    #endif
+} cudaField;
+
+typedef struct cudaField_int_ {
+    int * ptr;
+    unsigned int pitch;
+    #ifdef __HIPCC__
+    unsigned int dummy0;
+    #endif
+} cudaField_int;
+
+typedef struct cudaSoA_{
+    REAL * ptr;
+    unsigned int pitch;
+    unsigned int length_Y;
+    unsigned int length_Z;
+    #ifdef __HIPCC__
+    unsigned int dummy0;
+    #endif
+} cudaSoA;
+
+
+typedef struct cudaFieldPack_{
+    REAL * ptr;
+} cudaFieldPack;
+
+
+extern uint32_t BlockDimX;
+extern uint32_t BlockDimY;
+extern uint32_t BlockDimZ;
+
+extern dim3 BlockDim_X;
+extern dim3 BlockDim_Y;
+extern dim3 BlockDim_Z;
+
+extern dim3 BlockDim;
+
+extern int MaxThreadsPerBlock;
+extern int MaxBlockDimX;
+extern int MaxBlockDimY;
+extern int MaxBlockDimZ;
+extern int MaxGridDimX;
+extern int MaxGridDimY;
+extern int MaxGridDimZ;
+
+
+void cuda_commen_init();
+
+
+typedef struct cudaJobPackage_
+{
+    dim3 start;
+    dim3 end;
+    #ifdef __cplusplus
+    __host__ __device__ cudaJobPackage_(){};
+    __host__ __device__ cudaJobPackage_(const dim3 & start_in , const dim3 & end_in):start(start_in),end(end_in){};
+    __host__ __device__ void setup(const dim3 & start_in , const dim3 & end_in){start = start_in ,end = end_in;};
+    #endif
+} cudaJobPackage;
+
+inline void jobsize(cudaJobPackage *job , dim3 *size){
+    size->x = job->end.x - job->start.x;
+    size->y = job->end.y - job->start.y;
+    size->z = job->end.z - job->start.z;
+}
+
+inline void job2size(cudaJobPackage *job , dim3 *size){
+    size->x = job->end.x;
+    size->y = job->end.y;
+    size->z = job->end.z;
+}
+inline void job2job(cudaJobPackage * job , cudaJobPackage * out){
+    out->start.x = job->start.x;
+    out->start.y = job->start.y;
+    out->start.z = job->start.z;
+    
+    out->end.x = job->end.x + job->start.x;
+    out->end.y = job->end.y + job->start.y;
+    out->end.z = job->end.z + job->start.z;
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif

head/cuda_utility.h

+#ifndef _CUDA_UTILITY_H
+#define _CUDA_UTILITY_H
+
+#include "cuda_commen.h"
+#include "config_parameters.h"
+
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void * malloc_me_d(unsigned int * pitch , unsigned int size_x , unsigned size_y , unsigned size_z);
+
+
+
+#define get_Field(Field , x,y,z,...) (*(Field.ptr + (__VA_ARGS__ + x + Field.pitch *(y + (z)*ny_d) ) ) )
+#define get_Field_LAP(Field , x,y,z,...) (*(Field.ptr + (__VA_ARGS__ + x + Field.pitch *(y + (z)*ny_2lap_d) ) ) )
+
+void new_cudaField( cudaField ** pField, unsigned int size_x , unsigned int size_y , unsigned int size_z );
+void delete_cudaField(cudaField * pField);
+
+void new_cudaField_int( cudaField_int ** pField, unsigned int size_x , unsigned int size_y , unsigned int size_z );
+void delete_cudaField_int(cudaField_int * pField);
+
+
+#define get_SoA(SoA , x,y,z , var,...) (*( SoA.ptr + (__VA_ARGS__ + x + SoA.pitch*(y + ny_d*(z+ (var)*nz_d)))))
+#define get_SoA_LAP(SoA , x,y,z , var,...) (*( SoA.ptr + (__VA_ARGS__ + x + SoA.pitch*(y + ny_2lap_d*(z+ (var)*nz_2lap_d)))))
+
+void new_cudaSoA( cudaSoA ** pSoA, unsigned int size_x , unsigned int size_y , unsigned int size_z );
+void delete_cudaSoA(cudaSoA * pSoA);
+
+void new_cudaFieldPack(cudaFieldPack ** pack , unsigned int size_x , unsigned int size_y , unsigned int size_z);
+void delete_cudaFieldPack(cudaFieldPack * pField);
+
+
+void cal_grid_block_dim(dim3 * pgrid , dim3 * pblock , unsigned int threadx , unsigned int thready , unsigned int threadz ,  unsigned int size_x , unsigned int size_y , unsigned int size_z);
+
+
+
+enum {
+    H2D = 0,
+    D2H = 1
+};
+void memcpy_All(REAL *hostPtr, REAL *devPtr, unsigned int pitch, int mode, unsigned int size_x, unsigned int size_y, unsigned int size_z);
+void memcpy_All_int(int *hostPtr, int *devPtr, unsigned int pitch, int mode, unsigned int size_x, unsigned int size_y, unsigned int size_z);
+void memcpy_inner(REAL *hostPtr, REAL *devPtr, unsigned int pitch, int mode, unsigned int size_x, unsigned int size_y, unsigned int size_z);
+void memcpy_bound_x(REAL *hostPtr, REAL *devPtr, unsigned int pitch, int mode, unsigned int size_x, unsigned int size_y, unsigned int size_z);
+void memcpy_bound_y(REAL *hostPtr, REAL *devPtr, unsigned int pitch, int mode, unsigned int size_x, unsigned int size_y, unsigned int size_z);
+void memcpy_bound_z(REAL *hostPtr, REAL *devPtr, unsigned int pitch, int mode, unsigned int size_x, unsigned int size_y, unsigned int size_z);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file

head/init.h

+#ifndef __INIT_H
+#define __INIT_H
+#include "parameters.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+#ifdef __HIPCC__
+void opencfd_para_init_dev();
+#endif
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file

head/io_warp.h

+#ifndef __IO_WARP_H
+#define __IO_WARP_H
+
+#include "stdio.h"
+
+#define FWRITE(ptr , size , num , stream) \
+    {   \
+        int _ENCODE_START = num*size;\
+        int _ENCODE_END = num*size;\
+        fwrite(&_ENCODE_START , sizeof(int) , 1 , stream);\
+        fwrite(ptr , size , num , stream);\
+        fwrite(&_ENCODE_END , sizeof(int) , 1 , stream);\
+    }
+    
+#define FREAD(ptr , size , num , stream) \
+    {   int tmp_buffer;\
+        fread(&tmp_buffer , sizeof(int) , 1 , stream);\
+        fread(ptr , size , num , stream);\
+        fread(&tmp_buffer , sizeof(int) , 1 , stream);\
+    }
+
+#endif
+    
\ No newline at end of file

head/parameters.h

+#ifndef __PARAMETER_H
+#define __PARAMETER_H
+#include "mpi.h"
+#include "pthread.h"
+#include "config_parameters.h"
+#include "OCFD_Schemes_hybrid_auto.h"
+#include "stdarg.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+// ------For Doubleprecision  (real*8)------------------------------------------------------------------
+extern int OCFD_REAL_KIND;
+extern MPI_Datatype OCFD_DATA_TYPE;   //double precison computing
+//  ------For Single precision (real*4)-----------------------
+// typedef float REAL;
+// int OCFD_REAL_KIND=4
+// MPI_Datatype OCFD_DATA_TYPE;             //  single precision computing
+// ===========  Parameters for MPI ==========================================================----------------
+ 
+
+
+// -----constant-----
+extern REAL Re , Pr , Ama , Gamma , Ref_T , epsl_SW , PI;
+extern REAL Cv , Cp , Tsb , amu_C0;
+extern REAL split_C1 , split_C3, tmp0;
+extern REAL vis_flux_init_c;
+
+// --------MPI-------------------
+extern int my_id,npx,npy,npz;  //全局即方向id
+extern int NPX0 , NPY0 , NPZ0; // proc number on each direction
+extern int ID_XP1,ID_XM1,ID_YP1,ID_YM1,ID_ZP1,ID_ZM1; //邻居全局id
+
+extern MPI_Status status;
+extern MPI_Comm MPI_COMM_X,MPI_COMM_Y,MPI_COMM_Z,MPI_COMM_XY,MPI_COMM_XZ,MPI_COMM_YZ;
+extern MPI_Datatype TYPE_LAPX1,TYPE_LAPY1,TYPE_LAPZ1,TYPE_LAPX2,TYPE_LAPY2,TYPE_LAPZ2;
+
+extern int *i_offset,*j_offset,*k_offset,*i_nn,*j_nn,*k_nn; //某个方向的分块信息
+extern int MSG_BLOCK_SIZE;
+
+extern unsigned int nx,ny,nz; // 某方向所处理的个数
+extern unsigned int NX_GLOBAL,NY_GLOBAL,NZ_GLOBAL;
+extern unsigned int nx_lap,ny_lap,nz_lap;
+extern unsigned int nx_2lap,ny_2lap,nz_2lap;
+
+extern int Stream_MODE; //Stream 模式
+extern int TEST;
+extern pthread_t* thread_handles;
+// --------------------------------------------------------------------------------------------------------
+extern REAL dt,end_time,tt;
+extern REAL cpu_time;
+extern int Istep , end_step;
+
+// -----------Analysis and Save------------------------------------------
+
+extern int OCFD_ANA_time_average; 
+extern int OCFD_ana_flatplate,  OCFD_ana_saveplaneYZ;
+extern int OCFD_ana_saveplaneXZ, OCFD_ana_savePoints;
+extern int OCFD_ana_saveplaneXY, OCFD_ana_saveblock;
+extern int OCFD_ana_getQ;
+
+extern int Kstep_save, Kstep_show,N_ana,*K_ana,*Kstep_ana,KRK;
+
+//------------Scheme invis and vis----------------------------------------
+extern int Scheme_invis_ID;
+extern int Scheme_vis_ID;
+
+//---------------------WENO_SYMBO_Limiter------------------------------------------
+extern REAL WENO_TV_Limiter;
+extern REAL WENO_TV_MAX;
+
+// -----------Boundary Condition and Initial Condition-----------------------------
+extern int Iperiodic[3], Jacbound[3], D0_bound[6];
+extern int Non_ref[6];
+
+extern int Y_Jacbound,Y_bound0;
+
+extern int OCFD_BC_Liftbody3d;
+extern int IBC_USER;
+extern int Init_stat;
+
+extern char IF_SYMMETRY;
+extern char IF_WITHLEADING;  // 0 不含头部， 1 含头部 
+extern char IFLAG_UPPERBOUNDARY; // 0 激波外； 1 激波
+extern REAL AOA,TW,EPSL_WALL,EPSL_UPPER,WALL_DIS_BEGIN,WALL_DIS_END;
+extern REAL Sin_AOA , Cos_AOA;
+
+extern REAL *BC_rpara, (*ANA_rpara)[100]; 
+extern int *BC_npara, (*ANA_npara)[100]; 
+
+
+// ------Filter----------------------------------------------- 
+extern int FILTER_1,FILTER_2;   // 1: 5-point filter, 2: 7-point wide-band filter
+extern int NF_max;   // maximum of filtering 
+extern int Filter_Fo9p, Filter_Fopt_shock;
+extern int NFiltering,(*Filter_para)[11];      //Filtering
+extern REAL (*Filter_rpara)[3];   // s0  (0<s0<1), rth
+extern char IF_Filter_X , IF_Filter_Y , IF_Filter_Z ;
+extern int fiter_judge_X, fiter_judge_Y, fiter_judge_Z;
+// --------------------------------------------------
+
+
+
+// ----------------------------------------------------------
+// Coordinate parameters 
+extern REAL hx,hy,hz;  //参考空间网格尺寸
+extern REAL *pAxx,*pAyy,*pAzz,*pAkx,*pAky,*pAkz,*pAix,*pAiy,*pAiz,*pAsx,*pAsy,*pAsz,*pAjac;  // 度量系数矩阵
+
+
+// calculate memory
+extern REAL *pAmu; // viscous 3d [nz][nt][nx]
+extern REAL *pd,*pu,*pv,*pw,*pT,*pP; //  [nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern REAL *pf,*pfn,*pdu; // [nz][ny][nx][5]
+
+// used in filtering
+extern REAL *pf_lap; // [nz+2*LAP][ny+2*LAP][nx+2*LAP][5]
+
+// used in invis jacobian , is part of ptmpa
+extern REAL *pfp; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern REAL *pfm; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern REAL *pcc; // [nz+2*LAP][ny+2*LAP][nx+2*LAP]
+// used in invis jacobian , is part of ptmpb
+extern REAL *pdfp , *pdfm; // [nz][ny][nx]
+
+// used in ana
+extern REAL *pQ , *pLamda2;// [nx][ny][nx]
+extern REAL *pdm, *pum, *pvm, *pwm, *pTm;//[nx][ny][nz]
+extern int average_IO;
+extern int Istep_average;
+extern REAL tt_average;
+
+// used in vis jacobian , is part of ptmpb
+extern REAL * pEv1,*pEv2,*pEv3,*pEv4;  // [nz+2*LAP][ny+2*LAP][nx+2*LAP]
+// used in vis jacobian , is part of ptmpb
+extern REAL *puk,*pui,*pus,*pvk,*pvi,*pvs,*pwk,*pwi,*pws,*pTk,*pTi,*pTs;  //[nz][ny][nx]
+// used in mecpy
+extern REAL *pack_send_x,* pack_recv_x;
+extern REAL *pack_send_y,* pack_recv_y;
+extern REAL *pack_send_z,* pack_recv_z;
+// used in boundary_compressible_conner***********************************************************
+extern int MZMAX, MTMAX, INLET_BOUNDARY, IFLAG_WALL_NOT_NORMAL;
+extern REAL EPSL, X_DIST_BEGIN, X_DIST_END, BETA;
+extern REAL X_WALL_BEGIN, X_UP_BOUNDARY_BEGIN;
+
+// used in SCHEME_HYBRIDAUTO ********************************************************************
+extern int IFLAG_HybridAuto;
+extern int HybridA_Stage, Patch_max;
+extern int IFLAG_mem;
+extern HybridAuto_TYPE HybridAuto;
+extern int *scheme_x, *scheme_y, *scheme_z;
+
+extern int IF_CHARTERIC;
+
+void read_parameters();
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+
+
+

head/parameters_d.h

+#ifndef __PARAMETERS_D_H
+#define __PARAMETERS_D_H
+
+#include "config_parameters.h"
+#include "cuda_commen.h"
+
+
+// -----constant-----
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+extern cudaStream_t Stream[15];
+extern cudaEvent_t  Event[15];
+
+extern __device__ __constant__ REAL Ama_d , Gamma_d , epsl_sw_d;
+extern __device__ __constant__ REAL Cv_d , Cp_d , Tsb_d , amu_C0_d;
+extern __device__ __constant__ REAL split_C1_d , split_C3_d;
+extern __device__ __constant__ REAL vis_flux_init_c_d;
+
+extern __device__ __constant__ unsigned int nx_d,ny_d,nz_d; // 某方向所处理的个数
+extern __device__ __constant__ unsigned int nx_lap_d,ny_lap_d,nz_lap_d; // 某方向所处理的个数
+extern __device__ __constant__ unsigned int nx_2lap_d,ny_2lap_d,nz_2lap_d; // 某方向所处理的个数
+
+extern __device__ __constant__ REAL dt_d;
+extern __device__ __constant__ REAL Sin_AOA_d , Cos_AOA_d;
+extern __device__ __constant__ REAL TW_d;
+
+//---------------------WENO_SYMBO_Limiter------------------------------------------
+extern __device__ __constant__ REAL WENO_TV_Limiter_d;
+extern __device__ __constant__ REAL WENO_TV_MAX_d;
+
+// ----------------------------------------------------------
+// Coordinate parameters 
+extern __device__ __constant__ REAL hx_d,hy_d,hz_d;  //参考空间网格尺寸
+
+extern  cudaField *pAxx_d,*pAyy_d,*pAzz_d,*pAkx_d,*pAky_d,*pAkz_d,*pAix_d,*pAiy_d,*pAiz_d,*pAsx_d,*pAsy_d,*pAsz_d,*pAjac_d;  // 度量系数矩阵
+
+
+// calculate memory
+extern cudaField *pAmu_d; // viscous 3d [nz][nt][nx]
+extern cudaField *pd_d,*pu_d,*pv_d,*pw_d,*pT_d,*pP_d; //  [nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern cudaSoA *pf_d,*pfn_d,*pdu_d; // [nz][ny][nx][5]
+
+// used in filtering
+extern cudaSoA *pf_lap_d; // [nz+2*LAP][ny+2*LAP][nx+2*LAP][5]
+
+// used in analysis
+extern cudaField *pdm_d, *pum_d, *pvm_d, *pwm_d, *pTm_d;//[nz][ny][nx]
+
+// used in invis jacobian , is part of ptmpa
+extern cudaSoA *pfp_x_d; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern cudaSoA *pfm_x_d; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+
+extern cudaSoA *pfp_y_d; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern cudaSoA *pfm_y_d; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+
+extern cudaSoA *pfp_z_d; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+extern cudaSoA *pfm_z_d; // [5][nz+2*LAP][ny+2*LAP][nx+2*LAP]
+
+extern cudaField *pcc_d; // [nz+2*LAP][ny+2*LAP][nx+2*LAP]
+// used in invis jacobian , is part of ptmpb
+extern cudaField *pdfp_d , *pdfm_d; // [nz][ny][nx]
+
+
+
+// used in vis jacobian , is part of ptmpb
+extern cudaField * pEv1_d,*pEv2_d,*pEv3_d,*pEv4_d;  // [nz+2*LAP][ny+2*LAP][nx+2*LAP]
+// used in vis jacobian , is part of ptmpb
+extern cudaField *puk_d,*pui_d,*pus_d,*pvk_d,*pvi_d,*pvs_d,*pwk_d,*pwi_d,*pws_d,*pTk_d,*pTi_d,*pTs_d;  //[nz][ny][nx]
+extern cudaField *vis_u_d,*vis_v_d,*vis_w_d,*vis_T_d;  //[nz][ny][nx]
+
+extern cudaField grad_P;
+extern cudaField *pPP_d;
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+
+
+

head/test.h

+#ifndef __TEST_H
+#define __TEST_H
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "parameters.h"
+void write_block_me(char * name , REAL * u , int nx , int ny , int nz);
+void write_block_me1(char * name , REAL * u , int nx , int ny , int nz);
+void write_block_me2(char * name , REAL * u , int nx , int ny , int nz);
+void write_block_me3(char * name , REAL * u , int nx , int ny , int nz);
+#endif

head/utility.h

+#ifndef __UTILITY_H
+#define __UTILITY_H
+#include <stdlib.h>
+#include <stdio.h>
+#include "parameters.h"
+#include "config_parameters.h"
+
+#define PTR2ARRAY2(ptr,nx,ny) (REAL(*)[ny][nx])(ptr)
+#define PTR2ARRAY3(ptr,nx,ny,nz) (REAL(*)[nz][ny][nx])(ptr)
+#define MAX(a,b) (a>b? a : b)
+#define MIN(a,b) (a<b? a : b)
+#define PROCIdx2Num(proc_i , proc_j , proc_k) (proc_i + proc_j*NPX0 + proc_k*NPX0*NPY0)
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+#define malloc_me_Host(p, size) malloc_me_Host_((void**)&p , size ,  __FUNCTION__ , __FILE__ ,__LINE__)
+void malloc_me_Host_(void **p, int size , const char * funname , const char * file , int line);
+
+#define malloc_me(size) malloc_me_(size , __FUNCTION__ , __FILE__ ,__LINE__)
+void * malloc_me_(int size , const char * funname , const char * file , int line);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

head_hip/OCFD_Comput_Jacobian3d.h

+#include "hip/hip_runtime.h"
+#ifndef __OCFD_COMPUTE_JACOBIAN3D_H
+#define __OCFD_COMPUTE_JACOBIAN3D_H
+#include "parameters.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void Init_Jacobian3d();
+void Comput_Jacobian3d();
+void comput_Jac3d();
+void boundary_Jac3d_Axx();
+void boundary_Jac3d_Liftbody_Ajac();
+
+void boundary_Jac3d_kernal_y_ramp_wall(REAL seta);
+void boundary_Jac3d_kernal_z_cone_wall(REAL seta1, REAL seta2);
+
+#if ((defined __NVCC__) || (defined __HIPCC__))
+#include "cuda_commen.h"
+__global__ void comput_Jac3d_kernal(
+    cudaField xi,cudaField xj,cudaField xk,cudaField yi,cudaField yj,cudaField yk,cudaField zi,cudaField zj,cudaField zk,cudaField Akx,
+    cudaField Aky,cudaField Akz,cudaField Aix,cudaField Aiy,cudaField Aiz,cudaField Asx,cudaField Asy,cudaField Asz,cudaField Ajac,
+    cudaJobPackage job);
+#endif
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file

head_hip/OCFD_IO.h

+#ifndef __OCFD_IO_H
+#define __OCFD_IO_H
+#include "stdio.h"
+#include "parameters.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void read_file(int Iflag_av , REAL * pd , REAL * pu , REAL * pv , REAL * pw , REAL * pT);
+void OCFD_save(int Iflag_av, int Istep_name , REAL * pd , REAL * pu , REAL * pv , REAL * pw , REAL * pT);
+void write_3d1(MPI_File  pfile, MPI_Offset offset, REAL * pU);
+void read_3d1(MPI_File pfile, MPI_Offset offset, REAL * pU);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

head_hip/OCFD_IO_mpi.h

+#ifndef __OCFD_IO_MPI_H
+#define __OCFD_IO_MPI_H
+#include "parameters.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+void write_2d_XY(FILE * file, int ka, int size_x, int size_y, int lap, int *pU, REAL *pU1);
+void write_2d_XYa(FILE * file, int ka, REAL *pU);
+void write_2d_YZa(FILE * file, int ia, REAL *pU);
+void write_2d_XZa(FILE * file, int ja, REAL *pU);
+void write_points(FILE * file, REAL * pU, int mpoints, int *ia, int *ja, int *ka);
+void read_3d(MPI_File file, MPI_Offset offset, REAL *pU);
+void write_3d(MPI_File file, MPI_Offset offset, REAL *pU);
+void write_blockdata(FILE * file, REAL * pU, int ib, int ie, int jb, int je, int kb, int ke);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

head_hip/OCFD_NS_Jacobian3d.h

+#include "hip/hip_runtime.h"
+#ifndef __OCFD_NS_JACOBIAN_H
+#define __OCFD_NS_JACOBIAN_H
+#include "parameters.h"
+#include "cuda_commen.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void du_invis_Jacobian3d_init(cudaJobPackage job_in, hipStream_t *stream);
+
+void du_invis_Jacobian3d_x(cudaJobPackage job_in, cudaSoA *fp, cudaSoA *fm, hipStream_t *stream);
+//void du_invis_Jacobian3d_x(cudaJobPackage job_in, cudaSoA *fp_x, cudaSoA *fm_x, cudaSoA *fp_y, cudaSoA *fm_y, cudaSoA *fp_z, cudaSoA *fm_z, hipStream_t *stream);
+void du_invis_Jacobian3d_y(cudaJobPackage job_in, cudaSoA *fp, cudaSoA *fm, hipStream_t *stream);
+void du_invis_Jacobian3d_z(cudaJobPackage job_in, cudaSoA *fp, cudaSoA *fm, hipStream_t *stream);
+
+
+void du_viscous_Jacobian3d_init(hipStream_t *stream);
+
+void du_viscous_Jacobian3d_x_init(hipStream_t *stream);
+void du_viscous_Jacobian3d_x_final(cudaJobPackage job_in, hipStream_t *stream);
+void du_viscous_Jacobian3d_y_init(hipStream_t *stream);
+void du_viscous_Jacobian3d_y_final(cudaJobPackage job_in, hipStream_t *stream);
+void du_viscous_Jacobian3d_z_init(hipStream_t *stream);
+void du_viscous_Jacobian3d_z_final(cudaJobPackage job_in, hipStream_t *stream);
+
+void boundary_symmetry_pole_vis_y(hipStream_t *stream);
+
+
+/* ============= */
+void vis_flux_ker_x_warp();
+void vis_flux_ker_y_warp();
+void vis_flux_ker_z_warp();
+void boundary_symmetry_pole_vis_y_warp();
+
+
+typedef struct vis_flux_
+{
+    cudaField uk;
+	cudaField vk;
+	cudaField wk;
+	cudaField ui;
+	cudaField vi;
+	cudaField wi;
+	cudaField us;
+	cudaField vs;
+	cudaField ws;
+
+	cudaField Tk;
+	cudaField Ti;
+	cudaField Ts;
+
+	cudaField Amu;
+
+	cudaField u;
+	cudaField v;
+	cudaField w;
+
+	cudaField Ax;
+	cudaField Ay;
+	cudaField Az;
+
+	cudaField Ajac;
+	cudaField Akx;
+	cudaField Aky;
+	cudaField Akz;
+	cudaField Aix;
+	cudaField Aiy;
+	cudaField Aiz;
+	cudaField Asx;
+	cudaField Asy;
+	cudaField Asz;
+
+	cudaField Ev1;
+	cudaField Ev2;
+	cudaField Ev3;
+	cudaField Ev4;
+} vis_flux;
+
+
+/* ===================== */
+/*#if ((defined __NVCC__) || (defined __HIPCC__))
+__global__ void vis_flux_ker(
+cudaField Akx,cudaField Aky,cudaField Akz,cudaField Ajac,
+cudaField s11,cudaField s12,cudaField s13,cudaField s22,cudaField s23,cudaField s33,cudaField E1,cudaField E2,cudaField E3,
+cudaField Ev1,cudaField Ev2,cudaField Ev3,cudaField Ev4,
+cudaJobPackage job);
+
+__global__ void vis_flux_init_ker(
+cudaField uk,cudaField vk,cudaField wk,cudaField Tk,cudaField ui,cudaField vi,cudaField wi,cudaField Ti,cudaField us,cudaField vs,cudaField ws,cudaField Ts,
+cudaField Akx,cudaField Aky,cudaField Akz,cudaField Aix,cudaField Aiy,cudaField Aiz,cudaField Asx,cudaField Asy,cudaField Asz,
+cudaField Amu,
+cudaField u,cudaField v,cudaField w,
+cudaField s11,cudaField s22,cudaField s33,cudaField s12,cudaField s13,cudaField s23,
+cudaField E1,cudaField E2,cudaField E3,
+cudaJobPackage job
+);
+
+__global__ void final_flux_ker(cudaSoA du , cudaField df1 , cudaField df2 , cudaField df3 , cudaField df4 , cudaField AJac, cudaJobPackage job);
+#endif
+*/
+#ifdef __cplusplus
+}
+#endif
+#endif

head_hip/OCFD_NS_Solver.h

+#ifndef __OCFD_NS_SOLVER_H
+#define __OCFD_NS_SOLVER_H
+
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void NS_solver_real();
+
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file

head_hip/OCFD_Schemes.h

+#include "hip/hip_runtime.h"
+#ifndef __OCFD_SCHEME_H
+#define __OCFD_SCHEME_H
+
+#include "parameters.h"
+#include "cuda_commen.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+    
+__device__ REAL minmod2(REAL x1, REAL x2);
+__device__ REAL minmod4(REAL x1, REAL x2, REAL x3, REAL x4);
+
+__global__ void OCFD_CD6_kernel(dim3 flagxyzb, cudaField pf, cudaField pfy, cudaJobPackage job);
+__global__ void OCFD_CD8_kernel(dim3 flagxyzb, cudaField pf, cudaField pfy, cudaJobPackage job);
+
+__global__ void OCFD_dx0_CD6_kernel(cudaField pf , cudaField pfx , cudaJobPackage job);
+__global__ void OCFD_dy0_CD6_kernel(cudaField pf , cudaField pfy , cudaJobPackage job);
+__global__ void OCFD_dz0_CD6_kernel(cudaField pf , cudaField pfz , cudaJobPackage job);
+
+__global__ void OCFD_dx0_CD8_kernel(cudaField pf , cudaField pfx , cudaJobPackage job);
+__global__ void OCFD_dy0_CD8_kernel(cudaField pf , cudaField pfy , cudaJobPackage job);
+__global__ void OCFD_dz0_CD8_kernel(cudaField pf , cudaField pfz , cudaJobPackage job);
+
+
+__device__ REAL OCFD_weno7_SYMBO_kernel_P(int WENO_LMT_FLAG, REAL *stencil);
+__device__ REAL OCFD_weno7_SYMBO_kernel_M(int WENO_LMT_FLAG, REAL *stencil);
+__global__ void OCFD_weno7_SYMBO_P_kernel(int WENO_LMT_FLAG, dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+__global__ void OCFD_weno7_SYMBO_M_kernel(int WENO_LMT_FLAG, dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+
+
+__device__ void aa(int a, ...);
+
+__device__ REAL OCFD_weno5_kernel_P(REAL *stencil);
+__device__ REAL OCFD_weno5_kernel_M(REAL *stencil);
+__global__ void OCFD_weno5_P_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+__global__ void OCFD_weno5_M_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+
+__device__ REAL OCFD_weno7_kernel_P(REAL *stencil);
+__device__ REAL OCFD_weno7_kernel_M(REAL *stencil);
+__global__ void OCFD_weno7_P_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+__global__ void OCFD_weno7_M_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+
+
+__device__ REAL OCFD_NND2_kernel_P(REAL *stencil);
+__device__ REAL OCFD_NND2_kernel_M(REAL *stencil);
+__global__ void OCFD_NND2_P_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+__global__ void OCFD_NND2_M_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+
+
+__device__ REAL OCFD_UP7_kernel_P(REAL *stencil);
+__device__ REAL OCFD_UP7_kernel_M(REAL *stencil);
+__global__ void OCFD_UP7_P_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+__global__ void OCFD_UP7_M_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+
+
+__device__ REAL OCFD_OMP6_kernel_P(int OMP6_FLAG, REAL *stencil);
+__device__ REAL OCFD_OMP6_kernel_M(int OMP6_FLAG, REAL *stencil);
+__global__ void OCFD_OMP6_P_kernel(int OMP6_FLAG, dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+__global__ void OCFD_OMP6_M_kernel(int OMP6_FLAG, dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaJobPackage job);
+
+
+__device__ REAL OCFD_weno5_kernel_P(REAL *stencil);
+__device__ REAL OCFD_weno5_kernel_M(REAL *stencil);
+
+
+__device__ int get_data_kernel(int flagxyz, dim3 *coords, cudaSoA f, int num, REAL *stencil, int ka1, int kb1, REAL *sort, cudaJobPackage job);
+//__device__ void put_du_p_kernel(dim3 flagxyzb, dim3 coords, REAL tmp_r, REAL tmp_l, cudaSoA du, REAL *stencil, int num, cudaField Ajac, cudaJobPackage job);
+__device__ void put_du_p_kernel(dim3 flagxyz, dim3 coords, REAL tmp_r, REAL tmp_l, cudaSoA du, int num, cudaField Ajac, cudaJobPackage job);
+__device__ void put_du_m_kernel(dim3 flagxyz, dim3 coords, REAL tmp_r, REAL tmp_l, cudaSoA du, int num, cudaField Ajac, cudaJobPackage job);
+//__device__ void put_du_m_kernel(dim3 flagxyzb, dim3 coords, REAL tmp_r, REAL tmp_l, cudaSoA du, REAL *stencil, int num, cudaField Ajac, cudaJobPackage job);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

head_hip/OCFD_Schemes_Choose.h

+#ifndef __OCFD_SCHEME_CHOOSE_H
+#define __OCFD_SCHEME_CHOOSE_H
+
+#include "parameters.h"
+#include "cuda_commen.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void OCFD_dx0(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+void OCFD_dx1(cudaSoA pf, cudaSoA pdu, cudaField Ajac, cudaField u, cudaField v, cudaField w, cudaField cc, 
+	cudaField Ax, cudaField Ay, cudaField Az, cudaJobPackage job_in, dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+void OCFD_dx2(cudaSoA pf, cudaSoA pdu, cudaField Ajac, cudaField u, cudaField v, cudaField w, cudaField cc, 
+	cudaField Ax, cudaField Ay, cudaField Az, cudaJobPackage job_in, dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+
+void OCFD_dy0(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+void OCFD_dy1(cudaSoA pf, cudaSoA pdu, cudaField Ajac, cudaField u, cudaField v, cudaField w, cudaField cc, 
+	cudaField Ax, cudaField Ay, cudaField Az, cudaJobPackage job_in, dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+void OCFD_dy2(cudaSoA pf, cudaSoA pdu, cudaField Ajac, cudaField u, cudaField v, cudaField w, cudaField cc, 
+	cudaField Ax, cudaField Ay, cudaField Az, cudaJobPackage job_in, dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+
+void OCFD_dz0(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+void OCFD_dz1(cudaSoA pf, cudaSoA pdu, cudaField Ajac, cudaField u, cudaField v, cudaField w, cudaField cc, 
+	cudaField Ax, cudaField Ay, cudaField Az, cudaJobPackage job_in, dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+void OCFD_dz2(cudaSoA pf, cudaSoA pdu, cudaField Ajac, cudaField u, cudaField v, cudaField w, cudaField cc, 
+	cudaField Ax, cudaField Ay, cudaField Az, cudaJobPackage job_in, dim3 blockdim_in, hipStream_t *stream, int boundl, int boundr);
+
+void OCFD_dx0_jac(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream, int bound);
+void OCFD_dy0_jac(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream, int bound);
+void OCFD_dz0_jac(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream, int bound);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

head_hip/OCFD_Schemes_hybrid_auto.h

+#include "hip/hip_runtime.h"
+#ifndef __OCFD_SCHEMES_HYBRID_AUTO_H
+#define __OCFD_SCHEMES_HYBRID_AUTO_H
+
+#include "parameters.h"
+#include "cuda_commen.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+typedef struct HybridAuto_TYPE_{ //used in SCHEME_HYBRIDAUTO
+    REAL *P_intvs;
+    cudaField_int *scheme_x;
+    cudaField_int *scheme_y;
+    cudaField_int *scheme_z;
+    int Num_Patch_zones;
+    int *zones; //[6][Patch_max]
+    REAL *Pa_zones;
+    int IF_Smooth_dp;
+    int Style; 
+
+} HybridAuto_TYPE;
+    
+void Set_Scheme_HybridAuto(hipStream_t *stream);
+void Comput_P(cudaField *d, cudaField *T, cudaField *P, hipStream_t *stream);
+void Comput_grad(cudaField *f, hipStream_t *stream);
+void Smoothing_dp(hipStream_t *stream);
+void Patch_zones(hipStream_t *stream);
+void Boundary_dp(hipStream_t *stream);
+void Comput_Scheme_point(hipStream_t *stream);
+void Comput_Scheme_point_Jameson(hipStream_t *stream);
+
+__global__ void add_kernel(REAL *g_odata, int g_odata_size);
+__device__ REAL warpReduce(REAL mySum);
+
+__device__ int get_Hyscheme_flag_p_kernel(int flagxyz, dim3 coords, cudaField_int scheme, cudaJobPackage job);
+__device__ int get_Hyscheme_flag_m_kernel(int flagxyz, dim3 coords, cudaField_int scheme, cudaJobPackage job);
+
+__global__ void OCFD_HybridAuto_P_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaField_int scheme, cudaJobPackage job);
+__global__ void OCFD_HybridAuto_M_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaField_int scheme, cudaJobPackage job);
+
+__global__ void OCFD_HybridAuto_P_Jameson_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaField_int scheme, cudaJobPackage job);
+__global__ void OCFD_HybridAuto_M_Jameson_kernel(dim3 flagxyzb, cudaSoA f, cudaSoA du, cudaField Ajac, cudaField_int scheme, cudaJobPackage job);
+
+void HybridAuto_scheme_IO();
+void HybridAuto_scheme_Proportion();
+void modify_NT(hipStream_t *stream);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

head_hip/OCFD_Stream.h

+#ifndef __OCFD_STREAM_H
+#define __OCFD_STREAM_H
+#include "hip/hip_runtime.h"
+#include "hip/hip_runtime.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void du_comput(int KRK);
+
+void opencfd_mem_init_Stream();
+void opencfd_mem_finalize_Stream();
+
+void* du_Jacobian3d_all(void* pthread_id);
+
+void* du_invis_Jacobian3d(void* pthread_id);
+void* du_invis_Jacobian3d_all(void* pthread_id);
+
+void* du_invis_Jacobian3d_outer_exchange(hipStream_t *stream);
+void* du_invis_Jacobian3d_outer_x(hipStream_t *stream);
+void* du_invis_Jacobian3d_outer_y(hipStream_t *stream);
+void* du_invis_Jacobian3d_outer_z(hipStream_t *stream);
+
+void* du_vis_Jacobian3d(void* pthread_id);
+void* du_vis_Jacobian3d_all(void* pthread_id);
+
+void* du_vis_Jacobian3d_inner_x(hipStream_t *stream);
+void* du_vis_Jacobian3d_inner_y(hipStream_t *stream);
+void* du_vis_Jacobian3d_inner_z(hipStream_t *stream);
+
+void* du_vis_Jacobian3d_outer_x(hipStream_t *stream);
+void* du_vis_Jacobian3d_outer_y(hipStream_t *stream);
+void* du_vis_Jacobian3d_outer_z(hipStream_t *stream);
+
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+

head_hip/OCFD_ana.h

+#ifndef __OCFD_ANA_H
+#define __OCFD_ANA_H
+#include "cuda_commen.h"
+#include "parameters.h"
+#include "parameters_d.h"
+#include "cuda_utility.h"
+#include "math.h"
+#include "OCFD_Schemes_hybrid_auto.h"
+#include "OCFD_Schemes_Choose.h"
+#include "OCFD_IO_mpi.h"
+#include "utility.h"
+#include "OCFD_IO.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+void ana_residual(cudaField PE_d, REAL *E0);
+void ana_Jac();
+void OCFD_ana(int style, int ID);
+void ana_NAN_and_NT();
+void init_time_average();
+
+void get_inner(cudaField x1, cudaField x2);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file

head_hip/OCFD_bound_Scheme.h

+#ifndef __OCFD_BOUND_SCHEME_H
+#define __OCFD_BOUND_SCHEME_H
+
+#include "cuda_commen.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+
+void OCFD_Dx0_bound(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream);
+void OCFD_Dy0_bound(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream);
+void OCFD_Dz0_bound(cudaField pf , cudaField pfx , cudaJobPackage job_in , dim3 blockdim_in, hipStream_t *stream);
+
+
+void OCFD_bound(dim3 *flagxyzb, int boundp, int boundm, cudaJobPackage job);
+
+__device__ int OCFD_D0bound_scheme_kernel(REAL* tmp, dim3 flagxyzb, dim3 coords, REAL *stencil, int ka1, cudaJobPackage job);
+
+__device__ int OCFD_bound_scheme_kernel_p(REAL* flag, dim3 flagxyzb, dim3 coords, REAL *stencil, int ka1, int kb1, cudaJobPackage job);
+__device__ int OCFD_bound_scheme_kernel_m(REAL* flag, dim3 flagxyzb, dim3 coords, REAL *stencil, int ka1, int kb1, cudaJobPackage job);
+
+#ifdef __cplusplus
+}
+#endif
+#endif