Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
tsoc
openmm
Commits
d0d8fe98
Commit
d0d8fe98
authored
Jul 07, 2016
by
peastman
Browse files
Implemented parameter derivatives in OpenCL platform
parent
77fe86e4
Changes
9
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
204 additions
and
17 deletions
+204
-17
platforms/opencl/include/OpenCLBondedUtilities.h
platforms/opencl/include/OpenCLBondedUtilities.h
+11
-1
platforms/opencl/include/OpenCLContext.h
platforms/opencl/include/OpenCLContext.h
+31
-1
platforms/opencl/include/OpenCLNonbondedUtilities.h
platforms/opencl/include/OpenCLNonbondedUtilities.h
+11
-1
platforms/opencl/src/OpenCLBondedUtilities.cpp
platforms/opencl/src/OpenCLBondedUtilities.cpp
+27
-2
platforms/opencl/src/OpenCLContext.cpp
platforms/opencl/src/OpenCLContext.cpp
+24
-4
platforms/opencl/src/OpenCLKernels.cpp
platforms/opencl/src/OpenCLKernels.cpp
+56
-2
platforms/opencl/src/OpenCLNonbondedUtilities.cpp
platforms/opencl/src/OpenCLNonbondedUtilities.cpp
+30
-1
platforms/opencl/src/kernels/customNonbonded.cl
platforms/opencl/src/kernels/customNonbonded.cl
+8
-5
platforms/opencl/src/kernels/nonbonded.cl
platforms/opencl/src/kernels/nonbonded.cl
+6
-0
No files found.
platforms/opencl/include/OpenCLBondedUtilities.h
View file @
d0d8fe98
...
@@ -9,7 +9,7 @@
...
@@ -9,7 +9,7 @@
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* *
* *
* Portions copyright (c) 2011-201
5
Stanford University and the Authors. *
* Portions copyright (c) 2011-201
6
Stanford University and the Authors. *
* Authors: Peter Eastman *
* Authors: Peter Eastman *
* Contributors: *
* Contributors: *
* *
* *
...
@@ -100,6 +100,15 @@ public:
...
@@ -100,6 +100,15 @@ public:
* refer to it by this name.
* refer to it by this name.
*/
*/
std
::
string
addArgument
(
cl
::
Memory
&
data
,
const
std
::
string
&
type
);
std
::
string
addArgument
(
cl
::
Memory
&
data
,
const
std
::
string
&
type
);
/**
* Register that the interaction kernel will be computing the derivative of the potential energy
* with respect to a parameter.
*
* @param param the name of the parameter
* @return the variable that will be used to accumulate the derivative. Any code you pass to addInteraction() should
* add its contributions to this variable.
*/
std
::
string
addEnergyParameterDerivative
(
const
std
::
string
&
param
);
/**
/**
* Add some OpenCL code that should be included in the program, before the start of the kernel.
* Add some OpenCL code that should be included in the program, before the start of the kernel.
* This can be used, for example, to define functions that will be called by the kernel.
* This can be used, for example, to define functions that will be called by the kernel.
...
@@ -137,6 +146,7 @@ private:
...
@@ -137,6 +146,7 @@ private:
std
::
vector
<
OpenCLArray
*>
atomIndices
;
std
::
vector
<
OpenCLArray
*>
atomIndices
;
std
::
vector
<
OpenCLArray
*>
bufferIndices
;
std
::
vector
<
OpenCLArray
*>
bufferIndices
;
std
::
vector
<
std
::
string
>
prefixCode
;
std
::
vector
<
std
::
string
>
prefixCode
;
std
::
vector
<
std
::
string
>
energyParameterDerivatives
;
int
numForceBuffers
,
maxBonds
,
allGroups
;
int
numForceBuffers
,
maxBonds
,
allGroups
;
bool
hasInitializedKernels
;
bool
hasInitializedKernels
;
};
};
...
...
platforms/opencl/include/OpenCLContext.h
View file @
d0d8fe98
...
@@ -9,7 +9,7 @@
...
@@ -9,7 +9,7 @@
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* *
* *
* Portions copyright (c) 2009-201
5
Stanford University and the Authors. *
* Portions copyright (c) 2009-201
6
Stanford University and the Authors. *
* Authors: Peter Eastman *
* Authors: Peter Eastman *
* Contributors: *
* Contributors: *
* *
* *
...
@@ -264,6 +264,12 @@ public:
...
@@ -264,6 +264,12 @@ public:
OpenCLArray
&
getEnergyBuffer
()
{
OpenCLArray
&
getEnergyBuffer
()
{
return
*
energyBuffer
;
return
*
energyBuffer
;
}
}
/**
* Get the array which contains the buffer in which derivatives of the energy with respect to parameters are computed.
*/
OpenCLArray
&
getEnergyParamDerivBuffer
()
{
return
*
energyParamDerivBuffer
;
}
/**
/**
* Get a pointer to a block of pinned memory that can be used for efficient transfers between host and device.
* Get a pointer to a block of pinned memory that can be used for efficient transfers between host and device.
* This is guaranteed to be at least as large as any of the arrays returned by methods of this class.
* This is guaranteed to be at least as large as any of the arrays returned by methods of this class.
...
@@ -659,6 +665,27 @@ public:
...
@@ -659,6 +665,27 @@ public:
std
::
vector
<
ForcePostComputation
*>&
getPostComputations
()
{
std
::
vector
<
ForcePostComputation
*>&
getPostComputations
()
{
return
postComputations
;
return
postComputations
;
}
}
/**
* Get the names of all parameters with respect to which energy derivatives are computed.
*/
const
std
::
vector
<
std
::
string
>&
getEnergyParamDerivNames
()
const
{
return
energyParamDerivNames
;
}
/**
* Get a workspace data structure used for accumulating the values of derivatives of the energy
* with respect to parameters.
*/
std
::
map
<
std
::
string
,
double
>&
getEnergyParamDerivWorkspace
()
{
return
energyParamDerivWorkspace
;
}
/**
* Register that the derivative of potential energy with respect to a context parameter
* will need to be calculated. If this is called multiple times for a single parameter,
* it is only added to the list once.
*
* @param param the name of the parameter to add
*/
void
addEnergyParameterDerivative
(
const
std
::
string
&
param
);
/**
/**
* Mark that the current molecule definitions (and hence the atom order) may be invalid.
* Mark that the current molecule definitions (and hence the atom order) may be invalid.
* This should be called whenever force field parameters change. It will cause the definitions
* This should be called whenever force field parameters change. It will cause the definitions
...
@@ -725,7 +752,10 @@ private:
...
@@ -725,7 +752,10 @@ private:
OpenCLArray
*
forceBuffers
;
OpenCLArray
*
forceBuffers
;
OpenCLArray
*
longForceBuffer
;
OpenCLArray
*
longForceBuffer
;
OpenCLArray
*
energyBuffer
;
OpenCLArray
*
energyBuffer
;
OpenCLArray
*
energyParamDerivBuffer
;
OpenCLArray
*
atomIndexDevice
;
OpenCLArray
*
atomIndexDevice
;
std
::
vector
<
std
::
string
>
energyParamDerivNames
;
std
::
map
<
std
::
string
,
double
>
energyParamDerivWorkspace
;
std
::
vector
<
int
>
atomIndex
;
std
::
vector
<
int
>
atomIndex
;
std
::
vector
<
cl
::
Memory
*>
autoclearBuffers
;
std
::
vector
<
cl
::
Memory
*>
autoclearBuffers
;
std
::
vector
<
int
>
autoclearBufferSizes
;
std
::
vector
<
int
>
autoclearBufferSizes
;
...
...
platforms/opencl/include/OpenCLNonbondedUtilities.h
View file @
d0d8fe98
...
@@ -9,7 +9,7 @@
...
@@ -9,7 +9,7 @@
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* *
* *
* Portions copyright (c) 2009-201
3
Stanford University and the Authors. *
* Portions copyright (c) 2009-201
6
Stanford University and the Authors. *
* Authors: Peter Eastman *
* Authors: Peter Eastman *
* Contributors: *
* Contributors: *
* *
* *
...
@@ -88,6 +88,15 @@ public:
...
@@ -88,6 +88,15 @@ public:
* Add an array (other than a per-atom parameter) that should be passed as an argument to the default interaction kernel.
* Add an array (other than a per-atom parameter) that should be passed as an argument to the default interaction kernel.
*/
*/
void
addArgument
(
const
ParameterInfo
&
parameter
);
void
addArgument
(
const
ParameterInfo
&
parameter
);
/**
* Register that the interaction kernel will be computing the derivative of the potential energy
* with respect to a parameter.
*
* @param param the name of the parameter
* @return the variable that will be used to accumulate the derivative. Any code you pass to addInteraction() should
* add its contributions to this variable.
*/
std
::
string
addEnergyParameterDerivative
(
const
std
::
string
&
param
);
/**
/**
* Specify the list of exclusions that an interaction outside the default kernel will depend on.
* Specify the list of exclusions that an interaction outside the default kernel will depend on.
*
*
...
@@ -287,6 +296,7 @@ private:
...
@@ -287,6 +296,7 @@ private:
std
::
vector
<
std
::
vector
<
int
>
>
atomExclusions
;
std
::
vector
<
std
::
vector
<
int
>
>
atomExclusions
;
std
::
vector
<
ParameterInfo
>
parameters
;
std
::
vector
<
ParameterInfo
>
parameters
;
std
::
vector
<
ParameterInfo
>
arguments
;
std
::
vector
<
ParameterInfo
>
arguments
;
std
::
vector
<
std
::
string
>
energyParameterDerivatives
;
std
::
map
<
int
,
double
>
groupCutoff
;
std
::
map
<
int
,
double
>
groupCutoff
;
std
::
map
<
int
,
std
::
string
>
groupKernelSource
;
std
::
map
<
int
,
std
::
string
>
groupKernelSource
;
double
lastCutoff
;
double
lastCutoff
;
...
...
platforms/opencl/src/OpenCLBondedUtilities.cpp
View file @
d0d8fe98
...
@@ -6,7 +6,7 @@
...
@@ -6,7 +6,7 @@
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* *
* *
* Portions copyright (c) 2011-201
5
Stanford University and the Authors. *
* Portions copyright (c) 2011-201
6
Stanford University and the Authors. *
* Authors: Peter Eastman *
* Authors: Peter Eastman *
* Contributors: *
* Contributors: *
* *
* *
...
@@ -56,12 +56,25 @@ void OpenCLBondedUtilities::addInteraction(const vector<vector<int> >& atoms, co
...
@@ -56,12 +56,25 @@ void OpenCLBondedUtilities::addInteraction(const vector<vector<int> >& atoms, co
}
}
}
}
std
::
string
OpenCLBondedUtilities
::
addArgument
(
cl
::
Memory
&
data
,
const
string
&
type
)
{
string
OpenCLBondedUtilities
::
addArgument
(
cl
::
Memory
&
data
,
const
string
&
type
)
{
arguments
.
push_back
(
&
data
);
arguments
.
push_back
(
&
data
);
argTypes
.
push_back
(
type
);
argTypes
.
push_back
(
type
);
return
"customArg"
+
context
.
intToString
(
arguments
.
size
());
return
"customArg"
+
context
.
intToString
(
arguments
.
size
());
}
}
string
OpenCLBondedUtilities
::
addEnergyParameterDerivative
(
const
string
&
param
)
{
// See if the parameter has already been added.
int
index
;
for
(
index
=
0
;
index
<
energyParameterDerivatives
.
size
();
index
++
)
if
(
param
==
energyParameterDerivatives
[
index
])
break
;
if
(
index
==
energyParameterDerivatives
.
size
())
energyParameterDerivatives
.
push_back
(
param
);
context
.
addEnergyParameterDerivative
(
param
);
return
string
(
"energyParamDeriv"
)
+
context
.
intToString
(
index
);
}
void
OpenCLBondedUtilities
::
addPrefixCode
(
const
string
&
source
)
{
void
OpenCLBondedUtilities
::
addPrefixCode
(
const
string
&
source
)
{
for
(
int
i
=
0
;
i
<
(
int
)
prefixCode
.
size
();
i
++
)
for
(
int
i
=
0
;
i
<
(
int
)
prefixCode
.
size
();
i
++
)
if
(
prefixCode
[
i
]
==
source
)
if
(
prefixCode
[
i
]
==
source
)
...
@@ -190,13 +203,23 @@ void OpenCLBondedUtilities::initialize(const System& system) {
...
@@ -190,13 +203,23 @@ void OpenCLBondedUtilities::initialize(const System& system) {
}
}
for
(
int
i
=
0
;
i
<
(
int
)
arguments
.
size
();
i
++
)
for
(
int
i
=
0
;
i
<
(
int
)
arguments
.
size
();
i
++
)
s
<<
", __global "
<<
argTypes
[
i
]
<<
"* customArg"
<<
(
i
+
1
);
s
<<
", __global "
<<
argTypes
[
i
]
<<
"* customArg"
<<
(
i
+
1
);
if
(
energyParameterDerivatives
.
size
()
>
0
)
s
<<
", __global mixed* energyParamDerivs"
;
s
<<
") {
\n
"
;
s
<<
") {
\n
"
;
s
<<
"mixed energy = 0;
\n
"
;
s
<<
"mixed energy = 0;
\n
"
;
for
(
int
i
=
0
;
i
<
energyParameterDerivatives
.
size
();
i
++
)
s
<<
"mixed energyParamDeriv"
<<
i
<<
" = 0;
\n
"
;
for
(
int
i
=
0
;
i
<
setSize
;
i
++
)
{
for
(
int
i
=
0
;
i
<
setSize
;
i
++
)
{
int
force
=
set
[
i
];
int
force
=
set
[
i
];
s
<<
createForceSource
(
i
,
forceAtoms
[
force
].
size
(),
forceAtoms
[
force
][
0
].
size
(),
forceGroup
[
force
],
forceSource
[
force
]);
s
<<
createForceSource
(
i
,
forceAtoms
[
force
].
size
(),
forceAtoms
[
force
][
0
].
size
(),
forceGroup
[
force
],
forceSource
[
force
]);
}
}
s
<<
"energyBuffer[get_global_id(0)] += energy;
\n
"
;
s
<<
"energyBuffer[get_global_id(0)] += energy;
\n
"
;
const
vector
<
string
>&
allParamDerivNames
=
context
.
getEnergyParamDerivNames
();
int
numDerivs
=
allParamDerivNames
.
size
();
for
(
int
i
=
0
;
i
<
energyParameterDerivatives
.
size
();
i
++
)
for
(
int
index
=
0
;
index
<
numDerivs
;
index
++
)
if
(
allParamDerivNames
[
index
]
==
energyParameterDerivatives
[
i
])
s
<<
"energyParamDerivs[get_global_id(0)*"
<<
numDerivs
<<
"+"
<<
i
<<
"] += energyParamDeriv"
<<
i
<<
";
\n
"
;
s
<<
"}
\n
"
;
s
<<
"}
\n
"
;
map
<
string
,
string
>
defines
;
map
<
string
,
string
>
defines
;
defines
[
"PADDED_NUM_ATOMS"
]
=
context
.
intToString
(
context
.
getPaddedNumAtoms
());
defines
[
"PADDED_NUM_ATOMS"
]
=
context
.
intToString
(
context
.
getPaddedNumAtoms
());
...
@@ -274,6 +297,8 @@ void OpenCLBondedUtilities::computeInteractions(int groups) {
...
@@ -274,6 +297,8 @@ void OpenCLBondedUtilities::computeInteractions(int groups) {
}
}
for
(
int
j
=
0
;
j
<
(
int
)
arguments
.
size
();
j
++
)
for
(
int
j
=
0
;
j
<
(
int
)
arguments
.
size
();
j
++
)
kernel
.
setArg
<
cl
::
Memory
>
(
index
++
,
*
arguments
[
j
]);
kernel
.
setArg
<
cl
::
Memory
>
(
index
++
,
*
arguments
[
j
]);
if
(
energyParameterDerivatives
.
size
()
>
0
)
kernel
.
setArg
<
cl
::
Memory
>
(
index
++
,
context
.
getEnergyParamDerivBuffer
().
getDeviceBuffer
());
}
}
}
}
for
(
int
i
=
0
;
i
<
(
int
)
kernels
.
size
();
i
++
)
{
for
(
int
i
=
0
;
i
<
(
int
)
kernels
.
size
();
i
++
)
{
...
...
platforms/opencl/src/OpenCLContext.cpp
View file @
d0d8fe98
...
@@ -69,7 +69,7 @@ static void CL_CALLBACK errorCallback(const char* errinfo, const void* private_i
...
@@ -69,7 +69,7 @@ static void CL_CALLBACK errorCallback(const char* errinfo, const void* private_i
OpenCLContext
::
OpenCLContext
(
const
System
&
system
,
int
platformIndex
,
int
deviceIndex
,
const
string
&
precision
,
OpenCLPlatform
::
PlatformData
&
platformData
)
:
OpenCLContext
::
OpenCLContext
(
const
System
&
system
,
int
platformIndex
,
int
deviceIndex
,
const
string
&
precision
,
OpenCLPlatform
::
PlatformData
&
platformData
)
:
system
(
system
),
time
(
0.0
),
platformData
(
platformData
),
stepCount
(
0
),
computeForceCount
(
0
),
stepsSinceReorder
(
99999
),
atomsWereReordered
(
false
),
posq
(
NULL
),
system
(
system
),
time
(
0.0
),
platformData
(
platformData
),
stepCount
(
0
),
computeForceCount
(
0
),
stepsSinceReorder
(
99999
),
atomsWereReordered
(
false
),
posq
(
NULL
),
posqCorrection
(
NULL
),
velm
(
NULL
),
forceBuffers
(
NULL
),
longForceBuffer
(
NULL
),
energyBuffer
(
NULL
),
atomIndexDevice
(
NULL
),
integration
(
NULL
),
posqCorrection
(
NULL
),
velm
(
NULL
),
forceBuffers
(
NULL
),
longForceBuffer
(
NULL
),
energyBuffer
(
NULL
),
energyParamDerivBuffer
(
NULL
),
atomIndexDevice
(
NULL
),
integration
(
NULL
),
expression
(
NULL
),
bonded
(
NULL
),
nonbonded
(
NULL
),
thread
(
NULL
)
{
expression
(
NULL
),
bonded
(
NULL
),
nonbonded
(
NULL
),
thread
(
NULL
)
{
if
(
precision
==
"single"
)
{
if
(
precision
==
"single"
)
{
useDoublePrecision
=
false
;
useDoublePrecision
=
false
;
...
@@ -435,6 +435,8 @@ OpenCLContext::~OpenCLContext() {
...
@@ -435,6 +435,8 @@ OpenCLContext::~OpenCLContext() {
delete
longForceBuffer
;
delete
longForceBuffer
;
if
(
energyBuffer
!=
NULL
)
if
(
energyBuffer
!=
NULL
)
delete
energyBuffer
;
delete
energyBuffer
;
if
(
energyParamDerivBuffer
!=
NULL
)
delete
energyParamDerivBuffer
;
if
(
atomIndexDevice
!=
NULL
)
if
(
atomIndexDevice
!=
NULL
)
delete
atomIndexDevice
;
delete
atomIndexDevice
;
if
(
integration
!=
NULL
)
if
(
integration
!=
NULL
)
...
@@ -455,15 +457,16 @@ void OpenCLContext::initialize() {
...
@@ -455,15 +457,16 @@ void OpenCLContext::initialize() {
numForceBuffers
=
std
::
max
(
numForceBuffers
,
bonded
->
getNumForceBuffers
());
numForceBuffers
=
std
::
max
(
numForceBuffers
,
bonded
->
getNumForceBuffers
());
for
(
int
i
=
0
;
i
<
(
int
)
forces
.
size
();
i
++
)
for
(
int
i
=
0
;
i
<
(
int
)
forces
.
size
();
i
++
)
numForceBuffers
=
std
::
max
(
numForceBuffers
,
forces
[
i
]
->
getRequiredForceBuffers
());
numForceBuffers
=
std
::
max
(
numForceBuffers
,
forces
[
i
]
->
getRequiredForceBuffers
());
int
energyBufferSize
=
max
(
numThreadBlocks
*
ThreadBlockSize
,
nonbonded
->
getNumEnergyBuffers
());
if
(
useDoublePrecision
)
{
if
(
useDoublePrecision
)
{
forceBuffers
=
OpenCLArray
::
create
<
mm_double4
>
(
*
this
,
paddedNumAtoms
*
numForceBuffers
,
"forceBuffers"
);
forceBuffers
=
OpenCLArray
::
create
<
mm_double4
>
(
*
this
,
paddedNumAtoms
*
numForceBuffers
,
"forceBuffers"
);
force
=
OpenCLArray
::
create
<
mm_double4
>
(
*
this
,
&
forceBuffers
->
getDeviceBuffer
(),
paddedNumAtoms
,
"force"
);
force
=
OpenCLArray
::
create
<
mm_double4
>
(
*
this
,
&
forceBuffers
->
getDeviceBuffer
(),
paddedNumAtoms
,
"force"
);
energyBuffer
=
OpenCLArray
::
create
<
cl_double
>
(
*
this
,
max
(
numThreadBlocks
*
ThreadBlockSize
,
nonbonded
->
getNumE
nergyBuffer
s
())
,
"energyBuffer"
);
energyBuffer
=
OpenCLArray
::
create
<
cl_double
>
(
*
this
,
e
nergyBuffer
Size
,
"energyBuffer"
);
}
}
else
{
else
{
forceBuffers
=
OpenCLArray
::
create
<
mm_float4
>
(
*
this
,
paddedNumAtoms
*
numForceBuffers
,
"forceBuffers"
);
forceBuffers
=
OpenCLArray
::
create
<
mm_float4
>
(
*
this
,
paddedNumAtoms
*
numForceBuffers
,
"forceBuffers"
);
force
=
OpenCLArray
::
create
<
mm_float4
>
(
*
this
,
&
forceBuffers
->
getDeviceBuffer
(),
paddedNumAtoms
,
"force"
);
force
=
OpenCLArray
::
create
<
mm_float4
>
(
*
this
,
&
forceBuffers
->
getDeviceBuffer
(),
paddedNumAtoms
,
"force"
);
energyBuffer
=
OpenCLArray
::
create
<
cl_double
>
(
*
this
,
max
(
numThreadBlocks
*
ThreadBlockSize
,
nonbonded
->
getNumE
nergyBuffer
s
())
,
"energyBuffer"
);
energyBuffer
=
OpenCLArray
::
create
<
cl_double
>
(
*
this
,
e
nergyBuffer
Size
,
"energyBuffer"
);
}
}
if
(
supports64BitGlobalAtomics
)
{
if
(
supports64BitGlobalAtomics
)
{
longForceBuffer
=
OpenCLArray
::
create
<
cl_long
>
(
*
this
,
3
*
paddedNumAtoms
,
"longForceBuffer"
);
longForceBuffer
=
OpenCLArray
::
create
<
cl_long
>
(
*
this
,
3
*
paddedNumAtoms
,
"longForceBuffer"
);
...
@@ -475,7 +478,15 @@ void OpenCLContext::initialize() {
...
@@ -475,7 +478,15 @@ void OpenCLContext::initialize() {
}
}
addAutoclearBuffer
(
*
forceBuffers
);
addAutoclearBuffer
(
*
forceBuffers
);
addAutoclearBuffer
(
*
energyBuffer
);
addAutoclearBuffer
(
*
energyBuffer
);
int
bufferBytes
=
max
(
velm
->
getSize
()
*
velm
->
getElementSize
(),
energyBuffer
->
getSize
()
*
energyBuffer
->
getElementSize
());
int
numEnergyParamDerivs
=
energyParamDerivNames
.
size
();
if
(
numEnergyParamDerivs
>
0
)
{
if
(
useDoublePrecision
||
useMixedPrecision
)
energyParamDerivBuffer
=
OpenCLArray
::
create
<
cl_double
>
(
*
this
,
numEnergyParamDerivs
*
energyBufferSize
,
"energyParamDerivBuffer"
);
else
energyParamDerivBuffer
=
OpenCLArray
::
create
<
cl_float
>
(
*
this
,
numEnergyParamDerivs
*
energyBufferSize
,
"energyParamDerivBuffer"
);
addAutoclearBuffer
(
*
energyParamDerivBuffer
);
}
int
bufferBytes
=
max
(
velm
->
getSize
()
*
velm
->
getElementSize
(),
energyBufferSize
*
energyBuffer
->
getElementSize
());
pinnedBuffer
=
new
cl
::
Buffer
(
context
,
CL_MEM_ALLOC_HOST_PTR
,
bufferBytes
);
pinnedBuffer
=
new
cl
::
Buffer
(
context
,
CL_MEM_ALLOC_HOST_PTR
,
bufferBytes
);
pinnedMemory
=
currentQueue
.
enqueueMapBuffer
(
*
pinnedBuffer
,
CL_TRUE
,
CL_MAP_READ
|
CL_MAP_WRITE
,
0
,
bufferBytes
);
pinnedMemory
=
currentQueue
.
enqueueMapBuffer
(
*
pinnedBuffer
,
CL_TRUE
,
CL_MAP_READ
|
CL_MAP_WRITE
,
0
,
bufferBytes
);
for
(
int
i
=
0
;
i
<
numAtoms
;
i
++
)
{
for
(
int
i
=
0
;
i
<
numAtoms
;
i
++
)
{
...
@@ -1229,6 +1240,15 @@ void OpenCLContext::addPostComputation(ForcePostComputation* computation) {
...
@@ -1229,6 +1240,15 @@ void OpenCLContext::addPostComputation(ForcePostComputation* computation) {
postComputations
.
push_back
(
computation
);
postComputations
.
push_back
(
computation
);
}
}
void
OpenCLContext
::
addEnergyParameterDerivative
(
const
string
&
param
)
{
// See if this parameter has already been registered.
for
(
int
i
=
0
;
i
<
energyParamDerivNames
.
size
();
i
++
)
if
(
param
==
energyParamDerivNames
[
i
])
return
;
energyParamDerivNames
.
push_back
(
param
);
}
struct
OpenCLContext
::
WorkThread
::
ThreadData
{
struct
OpenCLContext
::
WorkThread
::
ThreadData
{
ThreadData
(
std
::
queue
<
OpenCLContext
::
WorkTask
*>&
tasks
,
bool
&
waiting
,
bool
&
finished
,
ThreadData
(
std
::
queue
<
OpenCLContext
::
WorkTask
*>&
tasks
,
bool
&
waiting
,
bool
&
finished
,
pthread_mutex_t
&
queueLock
,
pthread_cond_t
&
waitForTaskCondition
,
pthread_cond_t
&
queueEmptyCondition
)
:
pthread_mutex_t
&
queueLock
,
pthread_cond_t
&
waitForTaskCondition
,
pthread_cond_t
&
queueEmptyCondition
)
:
...
...
platforms/opencl/src/OpenCLKernels.cpp
View file @
d0d8fe98
...
@@ -125,6 +125,9 @@ void OpenCLCalcForcesAndEnergyKernel::beginComputation(ContextImpl& context, boo
...
@@ -125,6 +125,9 @@ void OpenCLCalcForcesAndEnergyKernel::beginComputation(ContextImpl& context, boo
OpenCLNonbondedUtilities& nb = cl.getNonbondedUtilities();
OpenCLNonbondedUtilities& nb = cl.getNonbondedUtilities();
cl.setComputeForceCount(cl.getComputeForceCount()+1);
cl.setComputeForceCount(cl.getComputeForceCount()+1);
nb.prepareInteractions(groups);
nb.prepareInteractions(groups);
map<string, double>& derivs = cl.getEnergyParamDerivWorkspace();
for (map<string, double>::const_iterator iter = context.getParameters().begin(); iter != context.getParameters().end(); ++iter)
derivs[iter->first] = 0;
}
}
double OpenCLCalcForcesAndEnergyKernel::finishComputation(ContextImpl& context, bool includeForces, bool includeEnergy, int groups, bool& valid) {
double OpenCLCalcForcesAndEnergyKernel::finishComputation(ContextImpl& context, bool includeForces, bool includeEnergy, int groups, bool& valid) {
...
@@ -369,7 +372,30 @@ void OpenCLUpdateStateDataKernel::getForces(ContextImpl& context, vector<Vec3>&
...
@@ -369,7 +372,30 @@ void OpenCLUpdateStateDataKernel::getForces(ContextImpl& context, vector<Vec3>&
}
}
void OpenCLUpdateStateDataKernel::getEnergyParameterDerivatives(ContextImpl& context, map<string, double>& derivs) {
void OpenCLUpdateStateDataKernel::getEnergyParameterDerivatives(ContextImpl& context, map<string, double>& derivs) {
const vector<string>& paramDerivNames = cl.getEnergyParamDerivNames();
int numDerivs = paramDerivNames.size();
if (numDerivs == 0)
return;
derivs = cl.getEnergyParamDerivWorkspace();
OpenCLArray& derivArray = cl.getEnergyParamDerivBuffer();
if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
vector<double> derivBuffers;
derivArray.download(derivBuffers);
for (int i = numDerivs; i < derivArray.getSize(); i += numDerivs)
for (int j = 0; j < numDerivs; j++)
derivBuffers[j] += derivBuffers[i+j];
for (int i = 0; i < numDerivs; i++)
derivs[paramDerivNames[i]] += derivBuffers[i];
}
else {
vector<float> derivBuffers;
derivArray.download(derivBuffers);
for (int i = numDerivs; i < derivArray.getSize(); i += numDerivs)
for (int j = 0; j < numDerivs; j++)
derivBuffers[j] += derivBuffers[i+j];
for (int i = 0; i < numDerivs; i++)
derivs[paramDerivNames[i]] += derivBuffers[i];
}
}
}
void OpenCLUpdateStateDataKernel::getPeriodicBoxVectors(ContextImpl& context, Vec3& a, Vec3& b, Vec3& c) const {
void OpenCLUpdateStateDataKernel::getPeriodicBoxVectors(ContextImpl& context, Vec3& a, Vec3& b, Vec3& c) const {
...
@@ -675,6 +701,12 @@ void OpenCLCalcCustomBondForceKernel::initialize(const System& system, const Cus
...
@@ -675,6 +701,12 @@ void OpenCLCalcCustomBondForceKernel::initialize(const System& system, const Cus
variables[name] = value;
variables[name] = value;
}
}
}
}
for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
string paramName = force.getEnergyParameterDerivativeName(i);
string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
expressions[derivVariable+" += "] = derivExpression;
}
stringstream compute;
stringstream compute;
for (int i = 0; i < (int) params->getBuffers().size(); i++) {
for (int i = 0; i < (int) params->getBuffers().size(); i++) {
const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
...
@@ -907,6 +939,12 @@ void OpenCLCalcCustomAngleForceKernel::initialize(const System& system, const Cu
...
@@ -907,6 +939,12 @@ void OpenCLCalcCustomAngleForceKernel::initialize(const System& system, const Cu
variables[name] = value;
variables[name] = value;
}
}
}
}
for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
string paramName = force.getEnergyParameterDerivativeName(i);
string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
expressions[derivVariable+" += "] = derivExpression;
}
stringstream compute;
stringstream compute;
for (int i = 0; i < (int) params->getBuffers().size(); i++) {
for (int i = 0; i < (int) params->getBuffers().size(); i++) {
const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
...
@@ -1359,6 +1397,12 @@ void OpenCLCalcCustomTorsionForceKernel::initialize(const System& system, const
...
@@ -1359,6 +1397,12 @@ void OpenCLCalcCustomTorsionForceKernel::initialize(const System& system, const
variables[name] = value;
variables[name] = value;
}
}
}
}
for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
string paramName = force.getEnergyParameterDerivativeName(i);
string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
expressions[derivVariable+" += "] = derivExpression;
}
stringstream compute;
stringstream compute;
for (int i = 0; i < (int) params->getBuffers().size(); i++) {
for (int i = 0; i < (int) params->getBuffers().size(); i++) {
const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
...
@@ -2307,6 +2351,12 @@ void OpenCLCalcCustomNonbondedForceKernel::initialize(const System& system, cons
...
@@ -2307,6 +2351,12 @@ void OpenCLCalcCustomNonbondedForceKernel::initialize(const System& system, cons
string value = "globals["+cl.intToString(i)+"]";
string value = "globals["+cl.intToString(i)+"]";
variables.push_back(makeVariable(name, prefix+value));
variables.push_back(makeVariable(name, prefix+value));
}
}
for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
string paramName = force.getEnergyParameterDerivativeName(i);
string derivVariable = cl.getNonbondedUtilities().addEnergyParameterDerivative(paramName);
Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
forceExpressions[derivVariable+" += interactionScale*switchValue*"] = derivExpression;
}
stringstream compute;
stringstream compute;
compute << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, prefix+"temp");
compute << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, prefix+"temp");
map<string, string> replacements;
map<string, string> replacements;
...
@@ -2639,7 +2689,11 @@ double OpenCLCalcCustomNonbondedForceKernel::execute(ContextImpl& context, bool
...
@@ -2639,7 +2689,11 @@ double OpenCLCalcCustomNonbondedForceKernel::execute(ContextImpl& context, bool
cl.executeKernel(interactionGroupKernel, numGroupThreadBlocks*forceThreadBlockSize, forceThreadBlockSize);
cl.executeKernel(interactionGroupKernel, numGroupThreadBlocks*forceThreadBlockSize, forceThreadBlockSize);
}
}
mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
return
longRangeCoefficient
/
(
boxSize
.
x
*
boxSize
.
y
*
boxSize
.
z
);
double volume = boxSize.x*boxSize.y*boxSize.z;
map<string, double>& derivs = cl.getEnergyParamDerivWorkspace();
for (int i = 0; i < longRangeCoefficientDerivs.size(); i++)
derivs[forceCopy->getEnergyParameterDerivativeName(i)] += longRangeCoefficientDerivs[i]/volume;
return longRangeCoefficient/volume;
}
}
void OpenCLCalcCustomNonbondedForceKernel::copyParametersToContext(ContextImpl& context, const CustomNonbondedForce& force) {
void OpenCLCalcCustomNonbondedForceKernel::copyParametersToContext(ContextImpl& context, const CustomNonbondedForce& force) {
...
...
platforms/opencl/src/OpenCLNonbondedUtilities.cpp
View file @
d0d8fe98
...
@@ -6,7 +6,7 @@
...
@@ -6,7 +6,7 @@
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Biological Structures at Stanford, funded under the NIH Roadmap for *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* Medical Research, grant U54 GM072970. See https://simtk.org. *
* *
* *
* Portions copyright (c) 2009-201
5
Stanford University and the Authors. *
* Portions copyright (c) 2009-201
6
Stanford University and the Authors. *
* Authors: Peter Eastman *
* Authors: Peter Eastman *
* Contributors: *
* Contributors: *
* *
* *
...
@@ -162,6 +162,19 @@ void OpenCLNonbondedUtilities::addArgument(const ParameterInfo& parameter) {
...
@@ -162,6 +162,19 @@ void OpenCLNonbondedUtilities::addArgument(const ParameterInfo& parameter) {
arguments
.
push_back
(
parameter
);
arguments
.
push_back
(
parameter
);
}
}
string
OpenCLNonbondedUtilities
::
addEnergyParameterDerivative
(
const
string
&
param
)
{
// See if the parameter has already been added.
int
index
;
for
(
index
=
0
;
index
<
energyParameterDerivatives
.
size
();
index
++
)
if
(
param
==
energyParameterDerivatives
[
index
])
break
;
if
(
index
==
energyParameterDerivatives
.
size
())
energyParameterDerivatives
.
push_back
(
param
);
context
.
addEnergyParameterDerivative
(
param
);
return
string
(
"energyParamDeriv"
)
+
context
.
intToString
(
index
);
}
void
OpenCLNonbondedUtilities
::
requestExclusions
(
const
vector
<
vector
<
int
>
>&
exclusionList
)
{
void
OpenCLNonbondedUtilities
::
requestExclusions
(
const
vector
<
vector
<
int
>
>&
exclusionList
)
{
if
(
anyExclusions
)
{
if
(
anyExclusions
)
{
bool
sameExclusions
=
(
exclusionList
.
size
()
==
atomExclusions
.
size
());
bool
sameExclusions
=
(
exclusionList
.
size
()
==
atomExclusions
.
size
());
...
@@ -591,6 +604,8 @@ cl::Kernel OpenCLNonbondedUtilities::createInteractionKernel(const string& sourc
...
@@ -591,6 +604,8 @@ cl::Kernel OpenCLNonbondedUtilities::createInteractionKernel(const string& sourc
args
<<
arguments
[
i
].
getName
();
args
<<
arguments
[
i
].
getName
();
}
}
}
}
if
(
energyParameterDerivatives
.
size
()
>
0
)
args
<<
", __global mixed* energyParamDerivs"
;
replacements
[
"PARAMETER_ARGUMENTS"
]
=
args
.
str
();
replacements
[
"PARAMETER_ARGUMENTS"
]
=
args
.
str
();
stringstream
loadLocal1
;
stringstream
loadLocal1
;
for
(
int
i
=
0
;
i
<
(
int
)
params
.
size
();
i
++
)
{
for
(
int
i
=
0
;
i
<
(
int
)
params
.
size
();
i
++
)
{
...
@@ -641,6 +656,18 @@ cl::Kernel OpenCLNonbondedUtilities::createInteractionKernel(const string& sourc
...
@@ -641,6 +656,18 @@ cl::Kernel OpenCLNonbondedUtilities::createInteractionKernel(const string& sourc
}
}
}
}
replacements
[
"LOAD_ATOM2_PARAMETERS"
]
=
load2j
.
str
();
replacements
[
"LOAD_ATOM2_PARAMETERS"
]
=
load2j
.
str
();
stringstream
initDerivs
;
for
(
int
i
=
0
;
i
<
energyParameterDerivatives
.
size
();
i
++
)
initDerivs
<<
"mixed energyParamDeriv"
<<
i
<<
" = 0;
\n
"
;
replacements
[
"INIT_DERIVATIVES"
]
=
initDerivs
.
str
();
stringstream
saveDerivs
;
const
vector
<
string
>&
allParamDerivNames
=
context
.
getEnergyParamDerivNames
();
int
numDerivs
=
allParamDerivNames
.
size
();
for
(
int
i
=
0
;
i
<
energyParameterDerivatives
.
size
();
i
++
)
for
(
int
index
=
0
;
index
<
numDerivs
;
index
++
)
if
(
allParamDerivNames
[
index
]
==
energyParameterDerivatives
[
i
])
saveDerivs
<<
"energyParamDerivs[get_global_id(0)*"
<<
numDerivs
<<
"+"
<<
i
<<
"] += energyParamDeriv"
<<
i
<<
";
\n
"
;
replacements
[
"SAVE_DERIVATIVES"
]
=
saveDerivs
.
str
();
map
<
string
,
string
>
defines
;
map
<
string
,
string
>
defines
;
if
(
useCutoff
)
if
(
useCutoff
)
defines
[
"USE_CUTOFF"
]
=
"1"
;
defines
[
"USE_CUTOFF"
]
=
"1"
;
...
@@ -716,5 +743,7 @@ cl::Kernel OpenCLNonbondedUtilities::createInteractionKernel(const string& sourc
...
@@ -716,5 +743,7 @@ cl::Kernel OpenCLNonbondedUtilities::createInteractionKernel(const string& sourc
for
(
int
i
=
0
;
i
<
(
int
)
arguments
.
size
();
i
++
)
{
for
(
int
i
=
0
;
i
<
(
int
)
arguments
.
size
();
i
++
)
{
kernel
.
setArg
<
cl
::
Memory
>
(
index
++
,
arguments
[
i
].
getMemory
());
kernel
.
setArg
<
cl
::
Memory
>
(
index
++
,
arguments
[
i
].
getMemory
());
}
}
if
(
energyParameterDerivatives
.
size
()
>
0
)
kernel
.
setArg
<
cl
::
Memory
>
(
index
++
,
context
.
getEnergyParamDerivBuffer
().
getDeviceBuffer
());
return
kernel
;
return
kernel
;
}
}
platforms/opencl/src/kernels/customNonbonded.cl
View file @
d0d8fe98
...
@@ -4,15 +4,18 @@ if (!isExcluded && r2 < CUTOFF_SQUARED) {
...
@@ -4,15 +4,18 @@ if (!isExcluded && r2 < CUTOFF_SQUARED) {
if
(
!isExcluded
)
{
if
(
!isExcluded
)
{
#
endif
#
endif
real
tempForce
=
0.0f
;
real
tempForce
=
0.0f
;
COMPUTE_FORCE
real
switchValue
=
1
,
switchDeriv
=
0
;
#
if
USE_SWITCH
#
if
USE_SWITCH
if
(
r
>
SWITCH_CUTOFF
)
{
if
(
r
>
SWITCH_CUTOFF
)
{
real
x
=
r-SWITCH_CUTOFF
;
real
x
=
r-SWITCH_CUTOFF
;
real
switchValue
=
1+x*x*x*
(
SWITCH_C3+x*
(
SWITCH_C4+x*SWITCH_C5
))
;
switchValue
=
1+x*x*x*
(
SWITCH_C3+x*
(
SWITCH_C4+x*SWITCH_C5
))
;
real
switchDeriv
=
x*x*
(
3*SWITCH_C3+x*
(
4*SWITCH_C4+x*5*SWITCH_C5
))
;
switchDeriv
=
x*x*
(
3*SWITCH_C3+x*
(
4*SWITCH_C4+x*5*SWITCH_C5
))
;
tempForce
=
tempForce*switchValue
-
tempEnergy*switchDeriv
;
tempEnergy
*=
switchValue
;
}
}
#
endif
COMPUTE_FORCE
#
if
USE_SWITCH
tempForce
=
tempForce*switchValue
-
tempEnergy*switchDeriv
;
tempEnergy
*=
switchValue
;
#
endif
#
endif
dEdR
+=
tempForce*invR
;
dEdR
+=
tempForce*invR
;
}
}
platforms/opencl/src/kernels/nonbonded.cl
View file @
d0d8fe98
...
@@ -35,6 +35,7 @@ __kernel void computeNonbonded(
...
@@ -35,6 +35,7 @@ __kernel void computeNonbonded(
const
unsigned
int
tgx
=
get_local_id
(
0
)
&
(
TILE_SIZE-1
)
;
const
unsigned
int
tgx
=
get_local_id
(
0
)
&
(
TILE_SIZE-1
)
;
const
unsigned
int
tbx
=
get_local_id
(
0
)
-
tgx
;
const
unsigned
int
tbx
=
get_local_id
(
0
)
-
tgx
;
mixed
energy
=
0
;
mixed
energy
=
0
;
INIT_DERIVATIVES
__local
AtomData
localData[FORCE_WORK_GROUP_SIZE]
;
__local
AtomData
localData[FORCE_WORK_GROUP_SIZE]
;
//
First
loop:
process
tiles
that
contain
exclusions.
//
First
loop:
process
tiles
that
contain
exclusions.
...
@@ -85,6 +86,7 @@ __kernel void computeNonbonded(
...
@@ -85,6 +86,7 @@ __kernel void computeNonbonded(
bool
isExcluded
=
(
atom1
>=
NUM_ATOMS
|
| atom2 >= NUM_ATOMS || !(excl & 0x1));
bool
isExcluded
=
(
atom1
>=
NUM_ATOMS
|
| atom2 >= NUM_ATOMS || !(excl & 0x1));
#endif
#endif
real tempEnergy = 0;
real tempEnergy = 0;
const real interactionScale = 0.5f;
COMPUTE_INTERACTION
COMPUTE_INTERACTION
energy += 0.5f*tempEnergy;
energy += 0.5f*tempEnergy;
#ifdef INCLUDE_FORCES
#ifdef INCLUDE_FORCES
...
@@ -144,6 +146,7 @@ __kernel void computeNonbonded(
...
@@ -144,6 +146,7 @@ __kernel void computeNonbonded(
bool
isExcluded
=
(
atom1
>=
NUM_ATOMS
|
| atom2 >= NUM_ATOMS || !(excl & 0x1));
bool
isExcluded
=
(
atom1
>=
NUM_ATOMS
|
| atom2 >= NUM_ATOMS || !(excl & 0x1));
#endif
#endif
real tempEnergy = 0;
real tempEnergy = 0;
const real interactionScale = 1.0f;
COMPUTE_INTERACTION
COMPUTE_INTERACTION
energy += tempEnergy;
energy += tempEnergy;
#ifdef INCLUDE_FORCES
#ifdef INCLUDE_FORCES
...
@@ -320,6 +323,7 @@ __kernel void computeNonbonded(
...
@@ -320,6 +323,7 @@ __kernel void computeNonbonded(
bool isExcluded = (atom1 >= NUM_ATOMS || atom2 >= NUM_ATOMS);
bool isExcluded = (atom1 >= NUM_ATOMS || atom2 >= NUM_ATOMS);
#endif
#endif
real tempEnergy = 0;
real tempEnergy = 0;
const real interactionScale = 1.0f;
COMPUTE_INTERACTION
COMPUTE_INTERACTION
energy += tempEnergy;
energy += tempEnergy;
#ifdef INCLUDE_FORCES
#ifdef INCLUDE_FORCES
...
@@ -374,6 +378,7 @@ __kernel void computeNonbonded(
...
@@ -374,6 +378,7 @@ __kernel void computeNonbonded(
bool isExcluded = (atom1 >= NUM_ATOMS |
|
atom2
>=
NUM_ATOMS
)
;
bool isExcluded = (atom1 >= NUM_ATOMS |
|
atom2
>=
NUM_ATOMS
)
;
#
endif
#
endif
real
tempEnergy
=
0
;
real
tempEnergy
=
0
;
const
real
interactionScale
=
1.0f
;
COMPUTE_INTERACTION
COMPUTE_INTERACTION
energy
+=
tempEnergy
;
energy
+=
tempEnergy
;
#
ifdef
INCLUDE_FORCES
#
ifdef
INCLUDE_FORCES
...
@@ -429,4 +434,5 @@ __kernel void computeNonbonded(
...
@@ -429,4 +434,5 @@ __kernel void computeNonbonded(
#
ifdef
INCLUDE_ENERGY
#
ifdef
INCLUDE_ENERGY
energyBuffer[get_global_id
(
0
)
]
+=
energy
;
energyBuffer[get_global_id
(
0
)
]
+=
energy
;
#
endif
#
endif
SAVE_DERIVATIVES
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment