OpenCLKernels.cpp 449 KB
Newer Older
1
2
3
4
5
6
7
8
/* -------------------------------------------------------------------------- *
 *                                   OpenMM                                   *
 * -------------------------------------------------------------------------- *
 * This is part of the OpenMM molecular simulation toolkit originating from   *
 * Simbios, the NIH National Center for Physics-Based Simulation of           *
 * Biological Structures at Stanford, funded under the NIH Roadmap for        *
 * Medical Research, grant U54 GM072970. See https://simtk.org.               *
 *                                                                            *
peastman's avatar
peastman committed
9
 * Portions copyright (c) 2008-2018 Stanford University and the Authors.      *
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 * Authors: Peter Eastman                                                     *
 * Contributors:                                                              *
 *                                                                            *
 * This program is free software: you can redistribute it and/or modify       *
 * it under the terms of the GNU Lesser General Public License as published   *
 * by the Free Software Foundation, either version 3 of the License, or       *
 * (at your option) any later version.                                        *
 *                                                                            *
 * This program is distributed in the hope that it will be useful,            *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
 * GNU Lesser General Public License for more details.                        *
 *                                                                            *
 * You should have received a copy of the GNU Lesser General Public License   *
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.      *
 * -------------------------------------------------------------------------- */

#include "OpenCLKernels.h"
28
#include "OpenCLForceInfo.h"
29
30
#include "openmm/LangevinIntegrator.h"
#include "openmm/Context.h"
31
#include "openmm/internal/AndersenThermostatImpl.h"
32
#include "openmm/internal/CMAPTorsionForceImpl.h"
33
#include "openmm/internal/ContextImpl.h"
34
#include "openmm/internal/CustomCentroidBondForceImpl.h"
35
#include "openmm/internal/CustomCompoundBondForceImpl.h"
36
#include "openmm/internal/CustomHbondForceImpl.h"
37
#include "openmm/internal/CustomManyParticleForceImpl.h"
38
#include "openmm/internal/CustomNonbondedForceImpl.h"
39
#include "openmm/internal/NonbondedForceImpl.h"
40
#include "openmm/internal/OSRngSeed.h"
Peter Eastman's avatar
Peter Eastman committed
41
#include "OpenCLBondedUtilities.h"
42
#include "OpenCLExpressionUtilities.h"
43
#include "OpenCLIntegrationUtilities.h"
44
#include "OpenCLNonbondedUtilities.h"
45
#include "OpenCLKernelSources.h"
46
#include "lepton/CustomFunction.h"
47
#include "lepton/ExpressionTreeNode.h"
48
#include "lepton/Operation.h"
49
50
#include "lepton/Parser.h"
#include "lepton/ParsedExpression.h"
51
#include "ReferenceTabulatedFunction.h"
52
53
#include "SimTKOpenMMRealType.h"
#include "SimTKOpenMMUtilities.h"
peastman's avatar
peastman committed
54
#include "jama_eig.h"
55
#include <algorithm>
56
#include <cmath>
57
#include <iterator>
58
#include <set>
59
60
61

using namespace OpenMM;
using namespace std;
62
using namespace Lepton;
63

64
65
66
67
68
69
70
static void setPosqCorrectionArg(OpenCLContext& cl, cl::Kernel& kernel, int index) {
    if (cl.getUseMixedPrecision())
        kernel.setArg<cl::Buffer>(index, cl.getPosqCorrection().getDeviceBuffer());
    else
        kernel.setArg<void*>(index, NULL);
}

71
72
73
74
75
76
77
static void setPeriodicBoxSizeArg(OpenCLContext& cl, cl::Kernel& kernel, int index) {
    if (cl.getUseDoublePrecision())
        kernel.setArg<mm_double4>(index, cl.getPeriodicBoxSizeDouble());
    else
        kernel.setArg<mm_float4>(index, cl.getPeriodicBoxSize());
}

78
static void setPeriodicBoxArgs(OpenCLContext& cl, cl::Kernel& kernel, int index) {
79
    if (cl.getUseDoublePrecision()) {
80
81
        kernel.setArg<mm_double4>(index++, cl.getPeriodicBoxSizeDouble());
        kernel.setArg<mm_double4>(index++, cl.getInvPeriodicBoxSizeDouble());
82
83
84
85
86
        kernel.setArg<mm_double4>(index++, cl.getPeriodicBoxVecXDouble());
        kernel.setArg<mm_double4>(index++, cl.getPeriodicBoxVecYDouble());
        kernel.setArg<mm_double4>(index, cl.getPeriodicBoxVecZDouble());
    }
    else {
87
88
        kernel.setArg<mm_float4>(index++, cl.getPeriodicBoxSize());
        kernel.setArg<mm_float4>(index++, cl.getInvPeriodicBoxSize());
89
90
91
92
        kernel.setArg<mm_float4>(index++, cl.getPeriodicBoxVecX());
        kernel.setArg<mm_float4>(index++, cl.getPeriodicBoxVecY());
        kernel.setArg<mm_float4>(index, cl.getPeriodicBoxVecZ());
    }
93
94
}

95
96
97
98
99
100
101
static bool isZeroExpression(const Lepton::ParsedExpression& expression) {
    const Lepton::Operation& op = expression.getRootNode().getOperation();
    if (op.getId() != Lepton::Operation::CONSTANT)
        return false;
    return (dynamic_cast<const Lepton::Operation::Constant&>(op).getValue() == 0.0);
}

102
103
104
105
static bool usesVariable(const Lepton::ExpressionTreeNode& node, const string& variable) {
    const Lepton::Operation& op = node.getOperation();
    if (op.getId() == Lepton::Operation::VARIABLE && op.getName() == variable)
        return true;
peastman's avatar
peastman committed
106
107
    for (auto& child : node.getChildren())
        if (usesVariable(child, variable))
108
109
110
111
112
113
114
115
            return true;
    return false;
}

static bool usesVariable(const Lepton::ParsedExpression& expression, const string& variable) {
    return usesVariable(expression.getRootNode(), variable);
}

116
117
118
119
static pair<ExpressionTreeNode, string> makeVariable(const string& name, const string& value) {
    return make_pair(ExpressionTreeNode(new Operation::Variable(name)), value);
}

120
void OpenCLCalcForcesAndEnergyKernel::initialize(const System& system) {
121
122
}

123
void OpenCLCalcForcesAndEnergyKernel::beginComputation(ContextImpl& context, bool includeForces, bool includeEnergy, int groups) {
124
    cl.setForcesValid(true);
125
    cl.clearAutoclearBuffers();
peastman's avatar
peastman committed
126
127
    for (auto computation : cl.getPreComputations())
        computation->computeForceAndEnergy(includeForces, includeEnergy, groups);
128
    OpenCLNonbondedUtilities& nb = cl.getNonbondedUtilities();
129
    cl.setComputeForceCount(cl.getComputeForceCount()+1);
130
    nb.prepareInteractions(groups);
131
    map<string, double>& derivs = cl.getEnergyParamDerivWorkspace();
peastman's avatar
peastman committed
132
133
    for (auto& param : context.getParameters())
        derivs[param.first] = 0;
134
135
}

136
double OpenCLCalcForcesAndEnergyKernel::finishComputation(ContextImpl& context, bool includeForces, bool includeEnergy, int groups, bool& valid) {
137
    cl.getBondedUtilities().computeInteractions(groups);
138
    cl.getNonbondedUtilities().computeInteractions(groups, includeForces, includeEnergy);
139
    double sum = 0.0;
peastman's avatar
peastman committed
140
141
    for (auto computation : cl.getPostComputations())
        sum += computation->computeForceAndEnergy(includeForces, includeEnergy, groups);
142
    cl.reduceForces();
143
    cl.getIntegrationUtilities().distributeForcesFromVirtualSites();
Peter Eastman's avatar
Peter Eastman committed
144
145
    if (includeEnergy)
        sum += cl.reduceEnergy();
146
147
    if (!cl.getForcesValid())
        valid = false;
148
    return sum;
149
150
}

151
void OpenCLUpdateStateDataKernel::initialize(const System& system) {
152
153
}

154
double OpenCLUpdateStateDataKernel::getTime(const ContextImpl& context) const {
155
    return cl.getTime();
156
157
}

158
void OpenCLUpdateStateDataKernel::setTime(ContextImpl& context, double time) {
159
    vector<OpenCLContext*>& contexts = cl.getPlatformData().contexts;
peastman's avatar
peastman committed
160
161
    for (auto ctx : contexts)
        ctx->setTime(time);
162
163
}

peastman's avatar
peastman committed
164
165
166
167
168
169
170
171
172
173
174
175
176
void OpenCLUpdateStateDataKernel::getPositions(ContextImpl& context, vector<Vec3>& positions) {
    int numParticles = context.getSystem().getNumParticles();
    positions.resize(numParticles);
    vector<mm_float4> posCorrection;
    if (cl.getUseDoublePrecision()) {
        mm_double4* posq = (mm_double4*) cl.getPinnedBuffer();
        cl.getPosq().download(posq);
    }
    else if (cl.getUseMixedPrecision()) {
        mm_float4* posq = (mm_float4*) cl.getPinnedBuffer();
        cl.getPosq().download(posq, false);
        posCorrection.resize(numParticles);
        cl.getPosqCorrection().download(posCorrection);
177
    }
peastman's avatar
peastman committed
178
179
180
    else {
        mm_float4* posq = (mm_float4*) cl.getPinnedBuffer();
        cl.getPosq().download(posq);
181
    }
peastman's avatar
peastman committed
182
183
184
185
    
    // Filling in the output array is done in parallel for speed.
    
    cl.getPlatformData().threads.execute([&] (ThreadPool& threads, int threadIndex) {
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        // Compute the position of each particle to return to the user.  This is done in parallel for speed.
        
        const vector<int>& order = cl.getAtomIndex();
        int numParticles = cl.getNumAtoms();
        Vec3 boxVectors[3];
        cl.getPeriodicBoxVectors(boxVectors[0], boxVectors[1], boxVectors[2]);
        int numThreads = threads.getNumThreads();
        int start = threadIndex*numParticles/numThreads;
        int end = (threadIndex+1)*numParticles/numThreads;
        if (cl.getUseDoublePrecision()) {
            mm_double4* posq = (mm_double4*) cl.getPinnedBuffer();
            for (int i = start; i < end; ++i) {
                mm_double4 pos = posq[i];
                mm_int4 offset = cl.getPosCellOffsets()[i];
                positions[order[i]] = Vec3(pos.x, pos.y, pos.z)-boxVectors[0]*offset.x-boxVectors[1]*offset.y-boxVectors[2]*offset.z;
            }
        }
        else if (cl.getUseMixedPrecision()) {
            mm_float4* posq = (mm_float4*) cl.getPinnedBuffer();
            for (int i = start; i < end; ++i) {
                mm_float4 pos1 = posq[i];
                mm_float4 pos2 = posCorrection[i];
                mm_int4 offset = cl.getPosCellOffsets()[i];
                positions[order[i]] = Vec3((double)pos1.x+(double)pos2.x, (double)pos1.y+(double)pos2.y, (double)pos1.z+(double)pos2.z)-boxVectors[0]*offset.x-boxVectors[1]*offset.y-boxVectors[2]*offset.z;
            }
        }
        else {
            mm_float4* posq = (mm_float4*) cl.getPinnedBuffer();
            for (int i = start; i < end; ++i) {
                mm_float4 pos = posq[i];
                mm_int4 offset = cl.getPosCellOffsets()[i];
                positions[order[i]] = Vec3(pos.x, pos.y, pos.z)-boxVectors[0]*offset.x-boxVectors[1]*offset.y-boxVectors[2]*offset.z;
            }
        }
peastman's avatar
peastman committed
220
    });
221
    cl.getPlatformData().threads.waitForThreads();
222
223
}

Peter Eastman's avatar
Peter Eastman committed
224
void OpenCLUpdateStateDataKernel::setPositions(ContextImpl& context, const vector<Vec3>& positions) {
225
    const vector<cl_int>& order = cl.getAtomIndex();
226
    int numParticles = context.getSystem().getNumParticles();
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
    if (cl.getUseDoublePrecision()) {
        mm_double4* posq = (mm_double4*) cl.getPinnedBuffer();
        cl.getPosq().download(posq);
        for (int i = 0; i < numParticles; ++i) {
            mm_double4& pos = posq[i];
            const Vec3& p = positions[order[i]];
            pos.x = p[0];
            pos.y = p[1];
            pos.z = p[2];
        }
        for (int i = numParticles; i < cl.getPaddedNumAtoms(); i++)
            posq[i] = mm_double4(0.0, 0.0, 0.0, 0.0);
        cl.getPosq().upload(posq);
    }
    else {
        mm_float4* posq = (mm_float4*) cl.getPinnedBuffer();
        cl.getPosq().download(posq);
        for (int i = 0; i < numParticles; ++i) {
            mm_float4& pos = posq[i];
            const Vec3& p = positions[order[i]];
            pos.x = (cl_float) p[0];
            pos.y = (cl_float) p[1];
            pos.z = (cl_float) p[2];
        }
        for (int i = numParticles; i < cl.getPaddedNumAtoms(); i++)
            posq[i] = mm_float4(0.0f, 0.0f, 0.0f, 0.0f);
        cl.getPosq().upload(posq);
    }
    if (cl.getUseMixedPrecision()) {
        mm_float4* posCorrection = (mm_float4*) cl.getPinnedBuffer();
        for (int i = 0; i < numParticles; ++i) {
            mm_float4& c = posCorrection[i];
            const Vec3& p = positions[order[i]];
            c.x = (cl_float) (p[0]-(cl_float)p[0]);
            c.y = (cl_float) (p[1]-(cl_float)p[1]);
            c.z = (cl_float) (p[2]-(cl_float)p[2]);
            c.w = 0;
        }
        for (int i = numParticles; i < cl.getPaddedNumAtoms(); i++)
            posCorrection[i] = mm_float4(0.0f, 0.0f, 0.0f, 0.0f);
        cl.getPosqCorrection().upload(posCorrection);
    }
peastman's avatar
peastman committed
269
270
    for (auto& offset : cl.getPosCellOffsets())
        offset = mm_int4(0, 0, 0, 0);
271
    cl.reorderAtoms();
272
273
}

Peter Eastman's avatar
Peter Eastman committed
274
void OpenCLUpdateStateDataKernel::getVelocities(ContextImpl& context, vector<Vec3>& velocities) {
275
    const vector<cl_int>& order = cl.getAtomIndex();
276
277
    int numParticles = context.getSystem().getNumParticles();
    velocities.resize(numParticles);
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
    if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
        mm_double4* velm = (mm_double4*) cl.getPinnedBuffer();
        cl.getVelm().download(velm);
        for (int i = 0; i < numParticles; ++i) {
            mm_double4 vel = velm[i];
            mm_int4 offset = cl.getPosCellOffsets()[i];
            velocities[order[i]] = Vec3(vel.x, vel.y, vel.z);
        }
    }
    else {
        mm_float4* velm = (mm_float4*) cl.getPinnedBuffer();
        cl.getVelm().download(velm);
        for (int i = 0; i < numParticles; ++i) {
            mm_float4 vel = velm[i];
            mm_int4 offset = cl.getPosCellOffsets()[i];
            velocities[order[i]] = Vec3(vel.x, vel.y, vel.z);
        }
295
296
297
    }
}

Peter Eastman's avatar
Peter Eastman committed
298
void OpenCLUpdateStateDataKernel::setVelocities(ContextImpl& context, const vector<Vec3>& velocities) {
299
    const vector<cl_int>& order = cl.getAtomIndex();
300
    int numParticles = context.getSystem().getNumParticles();
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
    if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
        mm_double4* velm = (mm_double4*) cl.getPinnedBuffer();
        cl.getVelm().download(velm);
        for (int i = 0; i < numParticles; ++i) {
            mm_double4& vel = velm[i];
            const Vec3& p = velocities[order[i]];
            vel.x = p[0];
            vel.y = p[1];
            vel.z = p[2];
        }
        for (int i = numParticles; i < cl.getPaddedNumAtoms(); i++)
            velm[i] = mm_double4(0.0, 0.0, 0.0, 0.0);
        cl.getVelm().upload(velm);
    }
    else {
        mm_float4* velm = (mm_float4*) cl.getPinnedBuffer();
        cl.getVelm().download(velm);
        for (int i = 0; i < numParticles; ++i) {
            mm_float4& vel = velm[i];
            const Vec3& p = velocities[order[i]];
            vel.x = p[0];
            vel.y = p[1];
            vel.z = p[2];
        }
        for (int i = numParticles; i < cl.getPaddedNumAtoms(); i++)
            velm[i] = mm_float4(0.0f, 0.0f, 0.0f, 0.0f);
        cl.getVelm().upload(velm);
    }
329
330
}

Peter Eastman's avatar
Peter Eastman committed
331
void OpenCLUpdateStateDataKernel::getForces(ContextImpl& context, vector<Vec3>& forces) {
332
    const vector<cl_int>& order = cl.getAtomIndex();
333
334
    int numParticles = context.getSystem().getNumParticles();
    forces.resize(numParticles);
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
    if (cl.getUseDoublePrecision()) {
        mm_double4* force = (mm_double4*) cl.getPinnedBuffer();
        cl.getForce().download(force);
        for (int i = 0; i < numParticles; ++i) {
            mm_double4 f = force[i];
            forces[order[i]] = Vec3(f.x, f.y, f.z);
        }
    }
    else {
        mm_float4* force = (mm_float4*) cl.getPinnedBuffer();
        cl.getForce().download(force);
        for (int i = 0; i < numParticles; ++i) {
            mm_float4 f = force[i];
            forces[order[i]] = Vec3(f.x, f.y, f.z);
        }
350
351
352
    }
}

353
void OpenCLUpdateStateDataKernel::getEnergyParameterDerivatives(ContextImpl& context, map<string, double>& derivs) {
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
    const vector<string>& paramDerivNames = cl.getEnergyParamDerivNames();
    int numDerivs = paramDerivNames.size();
    if (numDerivs == 0)
        return;
    derivs = cl.getEnergyParamDerivWorkspace();
    OpenCLArray& derivArray = cl.getEnergyParamDerivBuffer();
    if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
        vector<double> derivBuffers;
        derivArray.download(derivBuffers);
        for (int i = numDerivs; i < derivArray.getSize(); i += numDerivs)
            for (int j = 0; j < numDerivs; j++)
                derivBuffers[j] += derivBuffers[i+j];
        for (int i = 0; i < numDerivs; i++)
            derivs[paramDerivNames[i]] += derivBuffers[i];
    }
    else {
        vector<float> derivBuffers;
        derivArray.download(derivBuffers);
        for (int i = numDerivs; i < derivArray.getSize(); i += numDerivs)
            for (int j = 0; j < numDerivs; j++)
                derivBuffers[j] += derivBuffers[i+j];
        for (int i = 0; i < numDerivs; i++)
            derivs[paramDerivNames[i]] += derivBuffers[i];
    }
378
379
}

380
void OpenCLUpdateStateDataKernel::getPeriodicBoxVectors(ContextImpl& context, Vec3& a, Vec3& b, Vec3& c) const {
381
    cl.getPeriodicBoxVectors(a, b, c);
382
383
}

384
void OpenCLUpdateStateDataKernel::setPeriodicBoxVectors(ContextImpl& context, const Vec3& a, const Vec3& b, const Vec3& c) {
385
    vector<OpenCLContext*>& contexts = cl.getPlatformData().contexts;
386
387
388
389
390

    // If any particles have been wrapped to the first periodic box, we need to unwrap them
    // to avoid changing their positions.

    vector<Vec3> positions;
peastman's avatar
peastman committed
391
    for (auto offset : cl.getPosCellOffsets()) {
392
393
394
395
396
397
398
399
        if (offset.x != 0 || offset.y != 0 || offset.z != 0) {
            getPositions(context, positions);
            break;
        }
    }
    
    // Update the vectors.

peastman's avatar
peastman committed
400
401
    for (auto ctx : contexts)
        ctx->setPeriodicBoxVectors(a, b, c);
402
403
    if (positions.size() > 0)
        setPositions(context, positions);
404
405
}

Peter Eastman's avatar
Peter Eastman committed
406
void OpenCLUpdateStateDataKernel::createCheckpoint(ContextImpl& context, ostream& stream) {
407
    int version = 2;
Peter Eastman's avatar
Peter Eastman committed
408
    stream.write((char*) &version, sizeof(int));
409
410
    int precision = (cl.getUseDoublePrecision() ? 2 : cl.getUseMixedPrecision() ? 1 : 0);
    stream.write((char*) &precision, sizeof(int));
Peter Eastman's avatar
Peter Eastman committed
411
412
    double time = cl.getTime();
    stream.write((char*) &time, sizeof(double));
Peter Eastman's avatar
Peter Eastman committed
413
414
    int stepCount = cl.getStepCount();
    stream.write((char*) &stepCount, sizeof(int));
415
416
    int stepsSinceReorder = cl.getStepsSinceReorder();
    stream.write((char*) &stepsSinceReorder, sizeof(int));
417
    char* buffer = (char*) cl.getPinnedBuffer();
418
419
420
421
422
423
424
425
    cl.getPosq().download(buffer);
    stream.write(buffer, cl.getPosq().getSize()*cl.getPosq().getElementSize());
    if (cl.getUseMixedPrecision()) {
        cl.getPosqCorrection().download(buffer);
        stream.write(buffer, cl.getPosqCorrection().getSize()*cl.getPosqCorrection().getElementSize());
    }
    cl.getVelm().download(buffer);
    stream.write(buffer, cl.getVelm().getSize()*cl.getVelm().getElementSize());
426
    stream.write((char*) &cl.getAtomIndex()[0], sizeof(cl_int)*cl.getAtomIndex().size());
Peter Eastman's avatar
Peter Eastman committed
427
    stream.write((char*) &cl.getPosCellOffsets()[0], sizeof(mm_int4)*cl.getPosCellOffsets().size());
428
429
430
    Vec3 boxVectors[3];
    cl.getPeriodicBoxVectors(boxVectors[0], boxVectors[1], boxVectors[2]);
    stream.write((char*) boxVectors, 3*sizeof(Vec3));
Peter Eastman's avatar
Peter Eastman committed
431
    cl.getIntegrationUtilities().createCheckpoint(stream);
Peter Eastman's avatar
Peter Eastman committed
432
    SimTKOpenMMUtilities::createCheckpoint(stream);
Peter Eastman's avatar
Peter Eastman committed
433
434
435
436
437
}

void OpenCLUpdateStateDataKernel::loadCheckpoint(ContextImpl& context, istream& stream) {
    int version;
    stream.read((char*) &version, sizeof(int));
438
    if (version != 2)
Peter Eastman's avatar
Peter Eastman committed
439
        throw OpenMMException("Checkpoint was created with a different version of OpenMM");
440
441
442
443
444
    int precision;
    stream.read((char*) &precision, sizeof(int));
    int expectedPrecision = (cl.getUseDoublePrecision() ? 2 : cl.getUseMixedPrecision() ? 1 : 0);
    if (precision != expectedPrecision)
        throw OpenMMException("Checkpoint was created with a different numeric precision");
Peter Eastman's avatar
Peter Eastman committed
445
446
    double time;
    stream.read((char*) &time, sizeof(double));
447
    int stepCount, stepsSinceReorder;
Peter Eastman's avatar
Peter Eastman committed
448
    stream.read((char*) &stepCount, sizeof(int));
449
    stream.read((char*) &stepsSinceReorder, sizeof(int));
Peter Eastman's avatar
Peter Eastman committed
450
    vector<OpenCLContext*>& contexts = cl.getPlatformData().contexts;
peastman's avatar
peastman committed
451
452
453
454
    for (auto ctx : contexts) {
        ctx->setTime(time);
        ctx->setStepCount(stepCount);
        ctx->setStepsSinceReorder(stepsSinceReorder);
Peter Eastman's avatar
Peter Eastman committed
455
    }
456
    char* buffer = (char*) cl.getPinnedBuffer();
457
    stream.read(buffer, cl.getPosq().getSize()*cl.getPosq().getElementSize());
458
    cl.getPosq().upload(buffer);
459
460
461
462
463
    if (cl.getUseMixedPrecision()) {
        stream.read(buffer, cl.getPosqCorrection().getSize()*cl.getPosqCorrection().getElementSize());
        cl.getPosqCorrection().upload(buffer);
    }
    stream.read(buffer, cl.getVelm().getSize()*cl.getVelm().getElementSize());
464
465
466
    cl.getVelm().upload(buffer);
    stream.read((char*) &cl.getAtomIndex()[0], sizeof(cl_int)*cl.getAtomIndex().size());
    cl.getAtomIndexArray().upload(cl.getAtomIndex());
Peter Eastman's avatar
Peter Eastman committed
467
    stream.read((char*) &cl.getPosCellOffsets()[0], sizeof(mm_int4)*cl.getPosCellOffsets().size());
468
469
    Vec3 boxVectors[3];
    stream.read((char*) &boxVectors, 3*sizeof(Vec3));
peastman's avatar
peastman committed
470
471
    for (auto ctx : contexts)
        ctx->setPeriodicBoxVectors(boxVectors[0], boxVectors[1], boxVectors[2]);
Peter Eastman's avatar
Peter Eastman committed
472
    cl.getIntegrationUtilities().loadCheckpoint(stream);
Peter Eastman's avatar
Peter Eastman committed
473
    SimTKOpenMMUtilities::loadCheckpoint(stream);
peastman's avatar
peastman committed
474
475
    for (auto listener : cl.getReorderListeners())
        listener->execute();
Peter Eastman's avatar
Peter Eastman committed
476
477
}

478
479
480
481
void OpenCLApplyConstraintsKernel::initialize(const System& system) {
}

void OpenCLApplyConstraintsKernel::apply(ContextImpl& context, double tol) {
482
483
484
    if (!hasInitializedKernel) {
        hasInitializedKernel = true;
        map<string, string> defines;
485
        defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
486
487
488
        cl::Program program = cl.createProgram(OpenCLKernelSources::constraints, defines);
        applyDeltasKernel = cl::Kernel(program, "applyPositionDeltas");
        applyDeltasKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
489
490
        setPosqCorrectionArg(cl, applyDeltasKernel, 1);
        applyDeltasKernel.setArg<cl::Buffer>(2, cl.getIntegrationUtilities().getPosDelta().getDeviceBuffer());
491
492
493
494
495
496
    }
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
    cl.clearBuffer(integration.getPosDelta());
    integration.applyConstraints(tol);
    cl.executeKernel(applyDeltasKernel, cl.getNumAtoms());
    integration.computeVirtualSites();
497
498
}

499
500
501
502
void OpenCLApplyConstraintsKernel::applyToVelocities(ContextImpl& context, double tol) {
    cl.getIntegrationUtilities().applyVelocityConstraints(tol);
}

503
504
505
506
507
508
509
void OpenCLVirtualSitesKernel::initialize(const System& system) {
}

void OpenCLVirtualSitesKernel::computePositions(ContextImpl& context) {
    cl.getIntegrationUtilities().computeVirtualSites();
}

510
class OpenCLCalcHarmonicBondForceKernel::ForceInfo : public OpenCLForceInfo {
511
public:
512
    ForceInfo(const HarmonicBondForce& force) : OpenCLForceInfo(0), force(force) {
513
514
515
516
    }
    int getNumParticleGroups() {
        return force.getNumBonds();
    }
Peter Eastman's avatar
Peter Eastman committed
517
    void getParticlesInGroup(int index, vector<int>& particles) {
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
        int particle1, particle2;
        double length, k;
        force.getBondParameters(index, particle1, particle2, length, k);
        particles.resize(2);
        particles[0] = particle1;
        particles[1] = particle2;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2;
        double length1, length2, k1, k2;
        force.getBondParameters(group1, particle1, particle2, length1, k1);
        force.getBondParameters(group2, particle1, particle2, length2, k2);
        return (length1 == length2 && k1 == k2);
    }
private:
    const HarmonicBondForce& force;
};

void OpenCLCalcHarmonicBondForceKernel::initialize(const System& system, const HarmonicBondForce& force) {
537
538
539
540
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumBonds()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumBonds()/numContexts;
    numBonds = endIndex-startIndex;
541
542
    if (numBonds == 0)
        return;
Peter Eastman's avatar
Peter Eastman committed
543
    vector<vector<int> > atoms(numBonds, vector<int>(2));
peastman's avatar
peastman committed
544
    params.initialize<mm_float2>(cl, numBonds, "bondParams");
545
546
547
    vector<mm_float2> paramVector(numBonds);
    for (int i = 0; i < numBonds; i++) {
        double length, k;
Peter Eastman's avatar
Peter Eastman committed
548
        force.getBondParameters(startIndex+i, atoms[i][0], atoms[i][1], length, k);
549
        paramVector[i] = mm_float2((cl_float) length, (cl_float) k);
550
    }
peastman's avatar
peastman committed
551
    params.upload(paramVector);
Peter Eastman's avatar
Peter Eastman committed
552
    map<string, string> replacements;
553
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
554
    replacements["COMPUTE_FORCE"] = OpenCLKernelSources::harmonicBondForce;
peastman's avatar
peastman committed
555
    replacements["PARAMS"] = cl.getBondedUtilities().addArgument(params.getDeviceBuffer(), "float2");
556
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::bondForce, replacements), force.getForceGroup());
557
558
    info = new ForceInfo(force);
    cl.addForce(info);
559
560
}

561
double OpenCLCalcHarmonicBondForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
562
563
    return 0.0;
}
564

565
566
567
568
569
570
void OpenCLCalcHarmonicBondForceKernel::copyParametersToContext(ContextImpl& context, const HarmonicBondForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumBonds()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumBonds()/numContexts;
    if (numBonds != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of bonds has changed");
571
572
    if (numBonds == 0)
        return;
573
574
575
576
577
578
579
580
581
582
    
    // Record the per-bond parameters.
    
    vector<mm_float2> paramVector(numBonds);
    for (int i = 0; i < numBonds; i++) {
        int atom1, atom2;
        double length, k;
        force.getBondParameters(startIndex+i, atom1, atom2, length, k);
        paramVector[i] = mm_float2((cl_float) length, (cl_float) k);
    }
peastman's avatar
peastman committed
583
    params.upload(paramVector);
584
585
586
    
    // Mark that the current reordering may be invalid.
    
587
    cl.invalidateMolecules(info);
588
589
}

590
class OpenCLCalcCustomBondForceKernel::ForceInfo : public OpenCLForceInfo {
591
public:
592
    ForceInfo(const CustomBondForce& force) : OpenCLForceInfo(0), force(force) {
593
594
595
596
    }
    int getNumParticleGroups() {
        return force.getNumBonds();
    }
Peter Eastman's avatar
Peter Eastman committed
597
    void getParticlesInGroup(int index, vector<int>& particles) {
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
        int particle1, particle2;
        vector<double> parameters;
        force.getBondParameters(index, particle1, particle2, parameters);
        particles.resize(2);
        particles[0] = particle1;
        particles[1] = particle2;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2;
        vector<double> parameters1, parameters2;
        force.getBondParameters(group1, particle1, particle2, parameters1);
        force.getBondParameters(group2, particle1, particle2, parameters2);
        for (int i = 0; i < (int) parameters1.size(); i++)
            if (parameters1[i] != parameters2[i])
                return false;
        return true;
    }
private:
    const CustomBondForce& force;
};

OpenCLCalcCustomBondForceKernel::~OpenCLCalcCustomBondForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomBondForceKernel::initialize(const System& system, const CustomBondForce& force) {
625
626
627
628
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumBonds()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumBonds()/numContexts;
    numBonds = endIndex-startIndex;
629
630
    if (numBonds == 0)
        return;
631
    vector<vector<int> > atoms(numBonds, vector<int>(2));
632
633
    params = new OpenCLParameterSet(cl, force.getNumPerBondParameters(), numBonds, "customBondParams");
    vector<vector<cl_float> > paramVector(numBonds);
634
635
    for (int i = 0; i < numBonds; i++) {
        vector<double> parameters;
636
        force.getBondParameters(startIndex+i, atoms[i][0], atoms[i][1], parameters);
637
        paramVector[i].resize(parameters.size());
638
        for (int j = 0; j < (int) parameters.size(); j++)
639
            paramVector[i][j] = (cl_float) parameters[j];
640
    }
641
    params->setParameterValues(paramVector);
642
643
    info = new ForceInfo(force);
    cl.addForce(info);
644
645
646
647
648
649
650
651
652
653
654
655
656

    // Record information for the expressions.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
    Lepton::ParsedExpression energyExpression = Lepton::Parser::parse(force.getEnergyFunction()).optimize();
    Lepton::ParsedExpression forceExpression = energyExpression.differentiate("r").optimize();
    map<string, Lepton::ParsedExpression> expressions;
    expressions["energy += "] = energyExpression;
657
    expressions["real dEdR = "] = forceExpression;
658
659
660
661
662
663
664

    // Create the kernels.

    map<string, string> variables;
    variables["r"] = "r";
    for (int i = 0; i < force.getNumPerBondParameters(); i++) {
        const string& name = force.getPerBondParameterName(i);
665
        variables[name] = "bondParams"+params->getParameterSuffix(i);
666
    }
667
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
668
669
670
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customBondGlobals", CL_MEM_READ_ONLY);
        globals.upload(globalParamValues);
        string argName = cl.getBondedUtilities().addArgument(globals.getDeviceBuffer(), "float");
671
672
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
673
            string value = argName+"["+cl.intToString(i)+"]";
674
675
            variables[name] = value;
        }
676
    }
677
678
679
680
681
682
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string paramName = force.getEnergyParameterDerivativeName(i);
        string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
        Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
        expressions[derivVariable+" += "] = derivExpression;
    }
683
    stringstream compute;
684
685
    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
686
687
        string argName = cl.getBondedUtilities().addArgument(buffer.getMemory(), buffer.getType());
        compute<<buffer.getType()<<" bondParams"<<(i+1)<<" = "<<argName<<"[index];\n";
688
    }
peastman's avatar
peastman committed
689
690
    vector<const TabulatedFunction*> functions;
    vector<pair<string, string> > functionNames;
691
    compute << cl.getExpressionUtilities().createExpressions(expressions, variables, functions, functionNames, "temp");
692
    map<string, string> replacements;
693
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
694
    replacements["COMPUTE_FORCE"] = compute.str();
695
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::bondForce, replacements), force.getForceGroup());
696
697
}

698
double OpenCLCalcCustomBondForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
peastman's avatar
peastman committed
699
    if (globals.isInitialized()) {
700
        bool changed = false;
701
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
702
703
704
705
706
707
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
708
            globals.upload(globalParamValues);
709
710
711
712
    }
    return 0.0;
}

713
714
715
716
717
718
void OpenCLCalcCustomBondForceKernel::copyParametersToContext(ContextImpl& context, const CustomBondForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumBonds()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumBonds()/numContexts;
    if (numBonds != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of bonds has changed");
719
720
    if (numBonds == 0)
        return;
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
    
    // Record the per-bond parameters.
    
    vector<vector<cl_float> > paramVector(numBonds);
    vector<double> parameters;
    for (int i = 0; i < numBonds; i++) {
        int atom1, atom2;
        force.getBondParameters(startIndex+i, atom1, atom2, parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
737
    cl.invalidateMolecules(info);
738
739
}

740
class OpenCLCalcHarmonicAngleForceKernel::ForceInfo : public OpenCLForceInfo {
741
public:
742
    ForceInfo(const HarmonicAngleForce& force) : OpenCLForceInfo(0), force(force) {
743
744
745
746
    }
    int getNumParticleGroups() {
        return force.getNumAngles();
    }
Peter Eastman's avatar
Peter Eastman committed
747
    void getParticlesInGroup(int index, vector<int>& particles) {
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
        int particle1, particle2, particle3;
        double angle, k;
        force.getAngleParameters(index, particle1, particle2, particle3, angle, k);
        particles.resize(3);
        particles[0] = particle1;
        particles[1] = particle2;
        particles[2] = particle3;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2, particle3;
        double angle1, angle2, k1, k2;
        force.getAngleParameters(group1, particle1, particle2, particle3, angle1, k1);
        force.getAngleParameters(group2, particle1, particle2, particle3, angle2, k2);
        return (angle1 == angle2 && k1 == k2);
    }
private:
    const HarmonicAngleForce& force;
};

void OpenCLCalcHarmonicAngleForceKernel::initialize(const System& system, const HarmonicAngleForce& force) {
768
769
770
771
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumAngles()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumAngles()/numContexts;
    numAngles = endIndex-startIndex;
772
773
    if (numAngles == 0)
        return;
Peter Eastman's avatar
Peter Eastman committed
774
    vector<vector<int> > atoms(numAngles, vector<int>(3));
peastman's avatar
peastman committed
775
    params.initialize<mm_float2>(cl, numAngles, "angleParams");
776
777
778
    vector<mm_float2> paramVector(numAngles);
    for (int i = 0; i < numAngles; i++) {
        double angle, k;
Peter Eastman's avatar
Peter Eastman committed
779
        force.getAngleParameters(startIndex+i, atoms[i][0], atoms[i][1], atoms[i][2], angle, k);
780
        paramVector[i] = mm_float2((cl_float) angle, (cl_float) k);
781
782

    }
peastman's avatar
peastman committed
783
    params.upload(paramVector);
Peter Eastman's avatar
Peter Eastman committed
784
    map<string, string> replacements;
785
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
786
    replacements["COMPUTE_FORCE"] = OpenCLKernelSources::harmonicAngleForce;
peastman's avatar
peastman committed
787
    replacements["PARAMS"] = cl.getBondedUtilities().addArgument(params.getDeviceBuffer(), "float2");
788
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::angleForce, replacements), force.getForceGroup());
789
790
    info = new ForceInfo(force);
    cl.addForce(info);
791
792
}

793
double OpenCLCalcHarmonicAngleForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
794
795
796
    return 0.0;
}

797
798
799
800
801
802
void OpenCLCalcHarmonicAngleForceKernel::copyParametersToContext(ContextImpl& context, const HarmonicAngleForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumAngles()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumAngles()/numContexts;
    if (numAngles != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of angles has changed");
803
804
    if (numAngles == 0)
        return;
805
806
807
808
809
810
811
812
813
814
    
    // Record the per-angle parameters.
    
    vector<mm_float2> paramVector(numAngles);
    for (int i = 0; i < numAngles; i++) {
        int atom1, atom2, atom3;
        double angle, k;
        force.getAngleParameters(startIndex+i, atom1, atom2, atom3, angle, k);
        paramVector[i] = mm_float2((cl_float) angle, (cl_float) k);
    }
peastman's avatar
peastman committed
815
    params.upload(paramVector);
816
817
818
    
    // Mark that the current reordering may be invalid.
    
819
    cl.invalidateMolecules(info);
820
821
}

822
class OpenCLCalcCustomAngleForceKernel::ForceInfo : public OpenCLForceInfo {
823
public:
824
    ForceInfo(const CustomAngleForce& force) : OpenCLForceInfo(0), force(force) {
825
826
827
828
    }
    int getNumParticleGroups() {
        return force.getNumAngles();
    }
Peter Eastman's avatar
Peter Eastman committed
829
    void getParticlesInGroup(int index, vector<int>& particles) {
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
        int particle1, particle2, particle3;
        vector<double> parameters;
        force.getAngleParameters(index, particle1, particle2, particle3, parameters);
        particles.resize(3);
        particles[0] = particle1;
        particles[1] = particle2;
        particles[2] = particle3;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2, particle3;
        vector<double> parameters1, parameters2;
        force.getAngleParameters(group1, particle1, particle2, particle3, parameters1);
        force.getAngleParameters(group2, particle1, particle2, particle3, parameters2);
        for (int i = 0; i < (int) parameters1.size(); i++)
            if (parameters1[i] != parameters2[i])
                return false;
        return true;
    }
private:
    const CustomAngleForce& force;
};

OpenCLCalcCustomAngleForceKernel::~OpenCLCalcCustomAngleForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomAngleForceKernel::initialize(const System& system, const CustomAngleForce& force) {
858
859
860
861
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumAngles()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumAngles()/numContexts;
    numAngles = endIndex-startIndex;
862
863
    if (numAngles == 0)
        return;
864
    vector<vector<int> > atoms(numAngles, vector<int>(3));
865
866
867
868
    params = new OpenCLParameterSet(cl, force.getNumPerAngleParameters(), numAngles, "customAngleParams");
    vector<vector<cl_float> > paramVector(numAngles);
    for (int i = 0; i < numAngles; i++) {
        vector<double> parameters;
869
        force.getAngleParameters(startIndex+i, atoms[i][0], atoms[i][1], atoms[i][2], parameters);
870
871
872
873
874
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
875
876
    info = new ForceInfo(force);
    cl.addForce(info);
877
878
879
880
881
882
883
884
885
886
887
888
889

    // Record information for the expressions.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
    Lepton::ParsedExpression energyExpression = Lepton::Parser::parse(force.getEnergyFunction()).optimize();
    Lepton::ParsedExpression forceExpression = energyExpression.differentiate("theta").optimize();
    map<string, Lepton::ParsedExpression> expressions;
    expressions["energy += "] = energyExpression;
890
    expressions["real dEdAngle = "] = forceExpression;
891
892
893
894
895
896
897
898
899

    // Create the kernels.

    map<string, string> variables;
    variables["theta"] = "theta";
    for (int i = 0; i < force.getNumPerAngleParameters(); i++) {
        const string& name = force.getPerAngleParameterName(i);
        variables[name] = "angleParams"+params->getParameterSuffix(i);
    }
900
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
901
902
903
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customAngleGlobals", CL_MEM_READ_ONLY);
        globals.upload(globalParamValues);
        string argName = cl.getBondedUtilities().addArgument(globals.getDeviceBuffer(), "float");
904
905
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
906
            string value = argName+"["+cl.intToString(i)+"]";
907
908
            variables[name] = value;
        }
909
    }
910
911
912
913
914
915
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string paramName = force.getEnergyParameterDerivativeName(i);
        string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
        Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
        expressions[derivVariable+" += "] = derivExpression;
    }
916
917
918
    stringstream compute;
    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
919
920
        string argName = cl.getBondedUtilities().addArgument(buffer.getMemory(), buffer.getType());
        compute<<buffer.getType()<<" angleParams"<<(i+1)<<" = "<<argName<<"[index];\n";
921
    }
peastman's avatar
peastman committed
922
923
    vector<const TabulatedFunction*> functions;
    vector<pair<string, string> > functionNames;
924
    compute << cl.getExpressionUtilities().createExpressions(expressions, variables, functions, functionNames, "temp");
925
    map<string, string> replacements;
926
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
927
    replacements["COMPUTE_FORCE"] = compute.str();
928
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::angleForce, replacements), force.getForceGroup());
929
930
}

931
double OpenCLCalcCustomAngleForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
peastman's avatar
peastman committed
932
    if (globals.isInitialized()) {
933
934
935
936
937
938
939
940
        bool changed = false;
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
941
            globals.upload(globalParamValues);
942
943
944
945
    }
    return 0.0;
}

946
947
948
949
950
951
void OpenCLCalcCustomAngleForceKernel::copyParametersToContext(ContextImpl& context, const CustomAngleForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumAngles()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumAngles()/numContexts;
    if (numAngles != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of angles has changed");
952
953
    if (numAngles == 0)
        return;
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
    
    // Record the per-angle parameters.
    
    vector<vector<cl_float> > paramVector(numAngles);
    vector<double> parameters;
    for (int i = 0; i < numAngles; i++) {
        int atom1, atom2, atom3;
        force.getAngleParameters(startIndex+i, atom1, atom2, atom3, parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
970
    cl.invalidateMolecules(info);
971
972
}

973
class OpenCLCalcPeriodicTorsionForceKernel::ForceInfo : public OpenCLForceInfo {
974
public:
975
    ForceInfo(const PeriodicTorsionForce& force) : OpenCLForceInfo(0), force(force) {
976
977
978
979
    }
    int getNumParticleGroups() {
        return force.getNumTorsions();
    }
Peter Eastman's avatar
Peter Eastman committed
980
    void getParticlesInGroup(int index, vector<int>& particles) {
981
982
983
984
985
986
987
988
989
990
991
992
993
        int particle1, particle2, particle3, particle4, periodicity;
        double phase, k;
        force.getTorsionParameters(index, particle1, particle2, particle3, particle4, periodicity, phase, k);
        particles.resize(4);
        particles[0] = particle1;
        particles[1] = particle2;
        particles[2] = particle3;
        particles[3] = particle4;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2, particle3, particle4, periodicity1, periodicity2;
        double phase1, phase2, k1, k2;
        force.getTorsionParameters(group1, particle1, particle2, particle3, particle4, periodicity1, phase1, k1);
994
        force.getTorsionParameters(group2, particle1, particle2, particle3, particle4, periodicity2, phase2, k2);
995
996
997
998
999
1000
1001
        return (periodicity1 == periodicity2 && phase1 == phase2 && k1 == k2);
    }
private:
    const PeriodicTorsionForce& force;
};

void OpenCLCalcPeriodicTorsionForceKernel::initialize(const System& system, const PeriodicTorsionForce& force) {
1002
1003
1004
1005
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    numTorsions = endIndex-startIndex;
1006
1007
    if (numTorsions == 0)
        return;
Peter Eastman's avatar
Peter Eastman committed
1008
    vector<vector<int> > atoms(numTorsions, vector<int>(4));
peastman's avatar
peastman committed
1009
    params.initialize<mm_float4>(cl, numTorsions, "periodicTorsionParams");
1010
1011
    vector<mm_float4> paramVector(numTorsions);
    for (int i = 0; i < numTorsions; i++) {
Peter Eastman's avatar
Peter Eastman committed
1012
        int periodicity;
1013
        double phase, k;
Peter Eastman's avatar
Peter Eastman committed
1014
        force.getTorsionParameters(startIndex+i, atoms[i][0], atoms[i][1], atoms[i][2], atoms[i][3], periodicity, phase, k);
1015
        paramVector[i] = mm_float4((cl_float) k, (cl_float) phase, (cl_float) periodicity, 0.0f);
1016
    }
peastman's avatar
peastman committed
1017
    params.upload(paramVector);
Peter Eastman's avatar
Peter Eastman committed
1018
    map<string, string> replacements;
1019
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
1020
    replacements["COMPUTE_FORCE"] = OpenCLKernelSources::periodicTorsionForce;
peastman's avatar
peastman committed
1021
    replacements["PARAMS"] = cl.getBondedUtilities().addArgument(params.getDeviceBuffer(), "float4");
1022
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::torsionForce, replacements), force.getForceGroup());
1023
1024
    info = new ForceInfo(force);
    cl.addForce(info);
1025
1026
}

1027
double OpenCLCalcPeriodicTorsionForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
1028
1029
1030
    return 0.0;
}

1031
1032
1033
1034
1035
1036
void OpenCLCalcPeriodicTorsionForceKernel::copyParametersToContext(ContextImpl& context, const PeriodicTorsionForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    if (numTorsions != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of torsions has changed");
1037
1038
    if (numTorsions == 0)
        return;
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
    
    // Record the per-torsion parameters.
    
    vector<mm_float4> paramVector(numTorsions);
    for (int i = 0; i < numTorsions; i++) {
        int atom1, atom2, atom3, atom4, periodicity;
        double phase, k;
        force.getTorsionParameters(startIndex+i, atom1, atom2, atom3, atom4, periodicity, phase, k);
        paramVector[i] = mm_float4((cl_float) k, (cl_float) phase, (cl_float) periodicity, 0.0f);
    }
peastman's avatar
peastman committed
1049
    params.upload(paramVector);
1050
1051
1052
    
    // Mark that the current reordering may be invalid.
    
1053
    cl.invalidateMolecules(info);
1054
1055
}

1056
class OpenCLCalcRBTorsionForceKernel::ForceInfo : public OpenCLForceInfo {
1057
public:
1058
    ForceInfo(const RBTorsionForce& force) : OpenCLForceInfo(0), force(force) {
1059
1060
1061
1062
    }
    int getNumParticleGroups() {
        return force.getNumTorsions();
    }
Peter Eastman's avatar
Peter Eastman committed
1063
    void getParticlesInGroup(int index, vector<int>& particles) {
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
        int particle1, particle2, particle3, particle4;
        double c0, c1, c2, c3, c4, c5;
        force.getTorsionParameters(index, particle1, particle2, particle3, particle4, c0, c1, c2, c3, c4, c5);
        particles.resize(4);
        particles[0] = particle1;
        particles[1] = particle2;
        particles[2] = particle3;
        particles[3] = particle4;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2, particle3, particle4;
        double c0a, c0b, c1a, c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b;
        force.getTorsionParameters(group1, particle1, particle2, particle3, particle4, c0a, c1a, c2a, c3a, c4a, c5a);
1077
        force.getTorsionParameters(group2, particle1, particle2, particle3, particle4, c0b, c1b, c2b, c3b, c4b, c5b);
1078
1079
1080
1081
1082
1083
1084
        return (c0a == c0b && c1a == c1b && c2a == c2b && c3a == c3b && c4a == c4b && c5a == c5b);
    }
private:
    const RBTorsionForce& force;
};

void OpenCLCalcRBTorsionForceKernel::initialize(const System& system, const RBTorsionForce& force) {
1085
1086
1087
1088
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    numTorsions = endIndex-startIndex;
1089
1090
    if (numTorsions == 0)
        return;
Peter Eastman's avatar
Peter Eastman committed
1091
    vector<vector<int> > atoms(numTorsions, vector<int>(4));
peastman's avatar
peastman committed
1092
    params.initialize<mm_float8>(cl, numTorsions, "rbTorsionParams");
1093
1094
1095
    vector<mm_float8> paramVector(numTorsions);
    for (int i = 0; i < numTorsions; i++) {
        double c0, c1, c2, c3, c4, c5;
Peter Eastman's avatar
Peter Eastman committed
1096
        force.getTorsionParameters(startIndex+i, atoms[i][0], atoms[i][1], atoms[i][2], atoms[i][3], c0, c1, c2, c3, c4, c5);
1097
        paramVector[i] = mm_float8((cl_float) c0, (cl_float) c1, (cl_float) c2, (cl_float) c3, (cl_float) c4, (cl_float) c5, 0.0f, 0.0f);
1098
1099

    }
peastman's avatar
peastman committed
1100
    params.upload(paramVector);
Peter Eastman's avatar
Peter Eastman committed
1101
    map<string, string> replacements;
1102
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
1103
    replacements["COMPUTE_FORCE"] = OpenCLKernelSources::rbTorsionForce;
peastman's avatar
peastman committed
1104
    replacements["PARAMS"] = cl.getBondedUtilities().addArgument(params.getDeviceBuffer(), "float8");
1105
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::torsionForce, replacements), force.getForceGroup());
1106
1107
    info = new ForceInfo(force);
    cl.addForce(info);
1108
1109
}

1110
double OpenCLCalcRBTorsionForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
1111
1112
1113
    return 0.0;
}

1114
1115
1116
1117
1118
1119
void OpenCLCalcRBTorsionForceKernel::copyParametersToContext(ContextImpl& context, const RBTorsionForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    if (numTorsions != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of torsions has changed");
1120
1121
    if (numTorsions == 0)
        return;
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
    
    // Record the per-torsion parameters.
    
    vector<mm_float8> paramVector(numTorsions);
    for (int i = 0; i < numTorsions; i++) {
        int atom1, atom2, atom3, atom4;
        double c0, c1, c2, c3, c4, c5;
        force.getTorsionParameters(startIndex+i, atom1, atom2, atom3, atom4, c0, c1, c2, c3, c4, c5);
        paramVector[i] = mm_float8((cl_float) c0, (cl_float) c1, (cl_float) c2, (cl_float) c3, (cl_float) c4, (cl_float) c5, 0.0f, 0.0f);
    }
peastman's avatar
peastman committed
1132
    params.upload(paramVector);
1133
1134
1135
    
    // Mark that the current reordering may be invalid.
    
1136
    cl.invalidateMolecules(info);
1137
1138
}

1139
class OpenCLCalcCMAPTorsionForceKernel::ForceInfo : public OpenCLForceInfo {
1140
public:
1141
    ForceInfo(const CMAPTorsionForce& force) : OpenCLForceInfo(0), force(force) {
1142
1143
1144
1145
    }
    int getNumParticleGroups() {
        return force.getNumTorsions();
    }
Peter Eastman's avatar
Peter Eastman committed
1146
    void getParticlesInGroup(int index, vector<int>& particles) {
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
        int map, a1, a2, a3, a4, b1, b2, b3, b4;
        force.getTorsionParameters(index, map, a1, a2, a3, a4, b1, b2, b3, b4);
        particles.resize(8);
        particles[0] = a1;
        particles[1] = a2;
        particles[2] = a3;
        particles[3] = a4;
        particles[4] = b1;
        particles[5] = b2;
        particles[6] = b3;
        particles[7] = b4;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int map1, map2, a1, a2, a3, a4, b1, b2, b3, b4;
        force.getTorsionParameters(group1, map1, a1, a2, a3, a4, b1, b2, b3, b4);
        force.getTorsionParameters(group2, map2, a1, a2, a3, a4, b1, b2, b3, b4);
        return (map1 == map2);
    }
private:
    const CMAPTorsionForce& force;
};

void OpenCLCalcCMAPTorsionForceKernel::initialize(const System& system, const CMAPTorsionForce& force) {
1170
1171
1172
1173
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    numTorsions = endIndex-startIndex;
1174
1175
1176
1177
    if (numTorsions == 0)
        return;
    int numMaps = force.getNumMaps();
    vector<mm_float4> coeffVec;
1178
    mapPositionsVec.resize(numMaps);
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
    vector<double> energy;
    vector<vector<double> > c;
    int currentPosition = 0;
    for (int i = 0; i < numMaps; i++) {
        int size;
        force.getMapParameters(i, size, energy);
        CMAPTorsionForceImpl::calcMapDerivatives(size, energy, c);
        mapPositionsVec[i] = mm_int2(currentPosition, size);
        currentPosition += 4*size*size;
        for (int j = 0; j < size*size; j++) {
1189
1190
1191
1192
            coeffVec.push_back(mm_float4((float) c[j][0], (float) c[j][1], (float) c[j][2], (float) c[j][3]));
            coeffVec.push_back(mm_float4((float) c[j][4], (float) c[j][5], (float) c[j][6], (float) c[j][7]));
            coeffVec.push_back(mm_float4((float) c[j][8], (float) c[j][9], (float) c[j][10], (float) c[j][11]));
            coeffVec.push_back(mm_float4((float) c[j][12], (float) c[j][13], (float) c[j][14], (float) c[j][15]));
1193
1194
        }
    }
1195
    vector<vector<int> > atoms(numTorsions, vector<int>(8));
1196
    vector<cl_int> torsionMapsVec(numTorsions);
1197
1198
    for (int i = 0; i < numTorsions; i++)
        force.getTorsionParameters(startIndex+i, torsionMapsVec[i], atoms[i][0], atoms[i][1], atoms[i][2], atoms[i][3], atoms[i][4], atoms[i][5], atoms[i][6], atoms[i][7]);
peastman's avatar
peastman committed
1199
1200
1201
1202
1203
1204
    coefficients.initialize<mm_float4>(cl, coeffVec.size(), "cmapTorsionCoefficients");
    mapPositions.initialize<mm_int2>(cl, numMaps, "cmapTorsionMapPositions");
    torsionMaps.initialize<cl_int>(cl, numTorsions, "cmapTorsionMaps");
    coefficients.upload(coeffVec);
    mapPositions.upload(mapPositionsVec);
    torsionMaps.upload(torsionMapsVec);
1205
    map<string, string> replacements;
1206
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
peastman's avatar
peastman committed
1207
1208
1209
    replacements["COEFF"] = cl.getBondedUtilities().addArgument(coefficients.getDeviceBuffer(), "float4");
    replacements["MAP_POS"] = cl.getBondedUtilities().addArgument(mapPositions.getDeviceBuffer(), "int2");
    replacements["MAPS"] = cl.getBondedUtilities().addArgument(torsionMaps.getDeviceBuffer(), "int");
1210
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::cmapTorsionForce, replacements), force.getForceGroup());
1211
1212
    info = new ForceInfo(force);
    cl.addForce(info);
1213
1214
}

1215
double OpenCLCalcCMAPTorsionForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
1216
1217
1218
    return 0.0;
}

1219
void OpenCLCalcCMAPTorsionForceKernel::copyParametersToContext(ContextImpl& context, const CMAPTorsionForce& force) {
1220
1221
1222
1223
1224
    int numMaps = force.getNumMaps();
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    numTorsions = endIndex-startIndex;
peastman's avatar
peastman committed
1225
    if (mapPositions.getSize() != numMaps)
1226
        throw OpenMMException("updateParametersInContext: The number of maps has changed");
peastman's avatar
peastman committed
1227
    if (torsionMaps.getSize() != numTorsions)
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
        throw OpenMMException("updateParametersInContext: The number of CMAP torsions has changed");

    // Update the maps.

    vector<mm_float4> coeffVec;
    vector<double> energy;
    vector<vector<double> > c;
    int currentPosition = 0;
    for (int i = 0; i < numMaps; i++) {
        int size;
        force.getMapParameters(i, size, energy);
        if (size != mapPositionsVec[i].y)
            throw OpenMMException("updateParametersInContext: The size of a map has changed");
        CMAPTorsionForceImpl::calcMapDerivatives(size, energy, c);
        currentPosition += 4*size*size;
        for (int j = 0; j < size*size; j++) {
            coeffVec.push_back(mm_float4((float) c[j][0], (float) c[j][1], (float) c[j][2], (float) c[j][3]));
            coeffVec.push_back(mm_float4((float) c[j][4], (float) c[j][5], (float) c[j][6], (float) c[j][7]));
            coeffVec.push_back(mm_float4((float) c[j][8], (float) c[j][9], (float) c[j][10], (float) c[j][11]));
            coeffVec.push_back(mm_float4((float) c[j][12], (float) c[j][13], (float) c[j][14], (float) c[j][15]));
        }
    }
peastman's avatar
peastman committed
1250
    coefficients.upload(coeffVec);
1251
1252
1253
1254
1255
1256
1257
1258

    // Update the indices.

    vector<int> torsionMapsVec(numTorsions);
    for (int i = 0; i < numTorsions; i++) {
        int index[8];
        force.getTorsionParameters(i, torsionMapsVec[i], index[0], index[1], index[2], index[3], index[4], index[5], index[6], index[7]);
    }
peastman's avatar
peastman committed
1259
    torsionMaps.upload(torsionMapsVec);
1260
1261
}

1262
class OpenCLCalcCustomTorsionForceKernel::ForceInfo : public OpenCLForceInfo {
1263
public:
1264
    ForceInfo(const CustomTorsionForce& force) : OpenCLForceInfo(0), force(force) {
1265
1266
1267
1268
    }
    int getNumParticleGroups() {
        return force.getNumTorsions();
    }
Peter Eastman's avatar
Peter Eastman committed
1269
    void getParticlesInGroup(int index, vector<int>& particles) {
1270
1271
1272
        int particle1, particle2, particle3, particle4;
        vector<double> parameters;
        force.getTorsionParameters(index, particle1, particle2, particle3, particle4, parameters);
Peter Eastman's avatar
Bug fix  
Peter Eastman committed
1273
        particles.resize(4);
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
        particles[0] = particle1;
        particles[1] = particle2;
        particles[2] = particle3;
        particles[3] = particle4;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2, particle3, particle4;
        vector<double> parameters1, parameters2;
        force.getTorsionParameters(group1, particle1, particle2, particle3, particle4, parameters1);
        force.getTorsionParameters(group2, particle1, particle2, particle3, particle4, parameters2);
        for (int i = 0; i < (int) parameters1.size(); i++)
            if (parameters1[i] != parameters2[i])
                return false;
        return true;
    }
private:
    const CustomTorsionForce& force;
};

OpenCLCalcCustomTorsionForceKernel::~OpenCLCalcCustomTorsionForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomTorsionForceKernel::initialize(const System& system, const CustomTorsionForce& force) {
1299
1300
1301
1302
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    numTorsions = endIndex-startIndex;
1303
1304
    if (numTorsions == 0)
        return;
1305
    vector<vector<int> > atoms(numTorsions, vector<int>(4));
1306
1307
1308
1309
    params = new OpenCLParameterSet(cl, force.getNumPerTorsionParameters(), numTorsions, "customTorsionParams");
    vector<vector<cl_float> > paramVector(numTorsions);
    for (int i = 0; i < numTorsions; i++) {
        vector<double> parameters;
1310
        force.getTorsionParameters(startIndex+i, atoms[i][0], atoms[i][1], atoms[i][2], atoms[i][3], parameters);
1311
1312
1313
1314
1315
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
1316
1317
    info = new ForceInfo(force);
    cl.addForce(info);
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330

    // Record information for the expressions.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
    Lepton::ParsedExpression energyExpression = Lepton::Parser::parse(force.getEnergyFunction()).optimize();
    Lepton::ParsedExpression forceExpression = energyExpression.differentiate("theta").optimize();
    map<string, Lepton::ParsedExpression> expressions;
    expressions["energy += "] = energyExpression;
1331
    expressions["real dEdAngle = "] = forceExpression;
1332
1333
1334
1335
1336
1337
1338
1339
1340

    // Create the kernels.

    map<string, string> variables;
    variables["theta"] = "theta";
    for (int i = 0; i < force.getNumPerTorsionParameters(); i++) {
        const string& name = force.getPerTorsionParameterName(i);
        variables[name] = "torsionParams"+params->getParameterSuffix(i);
    }
1341
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
1342
1343
1344
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customTorsionGlobals", CL_MEM_READ_ONLY);
        globals.upload(globalParamValues);
        string argName = cl.getBondedUtilities().addArgument(globals.getDeviceBuffer(), "float");
1345
1346
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
1347
            string value = argName+"["+cl.intToString(i)+"]";
1348
1349
            variables[name] = value;
        }
1350
    }
1351
1352
1353
1354
1355
1356
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string paramName = force.getEnergyParameterDerivativeName(i);
        string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
        Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
        expressions[derivVariable+" += "] = derivExpression;
    }
1357
1358
1359
    stringstream compute;
    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
1360
1361
        string argName = cl.getBondedUtilities().addArgument(buffer.getMemory(), buffer.getType());
        compute<<buffer.getType()<<" torsionParams"<<(i+1)<<" = "<<argName<<"[index];\n";
1362
    }
peastman's avatar
peastman committed
1363
1364
    vector<const TabulatedFunction*> functions;
    vector<pair<string, string> > functionNames;
1365
    compute << cl.getExpressionUtilities().createExpressions(expressions, variables, functions, functionNames, "temp");
1366
    map<string, string> replacements;
1367
    replacements["APPLY_PERIODIC"] = (force.usesPeriodicBoundaryConditions() ? "1" : "0");
1368
    replacements["COMPUTE_FORCE"] = compute.str();
1369
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::torsionForce, replacements), force.getForceGroup());
1370
1371
}

1372
double OpenCLCalcCustomTorsionForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
peastman's avatar
peastman committed
1373
    if (globals.isInitialized()) {
1374
1375
1376
1377
1378
1379
1380
1381
        bool changed = false;
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
1382
            globals.upload(globalParamValues);
1383
1384
1385
1386
    }
    return 0.0;
}

1387
1388
1389
1390
1391
1392
void OpenCLCalcCustomTorsionForceKernel::copyParametersToContext(ContextImpl& context, const CustomTorsionForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumTorsions()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumTorsions()/numContexts;
    if (numTorsions != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of torsions has changed");
1393
1394
    if (numTorsions == 0)
        return;
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
    
    // Record the per-torsion parameters.
    
    vector<vector<cl_float> > paramVector(numTorsions);
    vector<double> parameters;
    for (int i = 0; i < numTorsions; i++) {
        int atom1, atom2, atom3, atom4;
        force.getTorsionParameters(startIndex+i, atom1, atom2, atom3, atom4, parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
1411
    cl.invalidateMolecules(info);
1412
1413
}

1414
class OpenCLCalcNonbondedForceKernel::ForceInfo : public OpenCLForceInfo {
1415
public:
1416
    ForceInfo(int requiredBuffers, const NonbondedForce& force) : OpenCLForceInfo(requiredBuffers), force(force) {
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        double charge1, charge2, sigma1, sigma2, epsilon1, epsilon2;
        force.getParticleParameters(particle1, charge1, sigma1, epsilon1);
        force.getParticleParameters(particle2, charge2, sigma2, epsilon2);
        return (charge1 == charge2 && sigma1 == sigma2 && epsilon1 == epsilon2);
    }
    int getNumParticleGroups() {
        return force.getNumExceptions();
    }
Peter Eastman's avatar
Peter Eastman committed
1427
    void getParticlesInGroup(int index, vector<int>& particles) {
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
        int particle1, particle2;
        double chargeProd, sigma, epsilon;
        force.getExceptionParameters(index, particle1, particle2, chargeProd, sigma, epsilon);
        particles.resize(2);
        particles[0] = particle1;
        particles[1] = particle2;
    }
    bool areGroupsIdentical(int group1, int group2) {
        int particle1, particle2;
        double chargeProd1, chargeProd2, sigma1, sigma2, epsilon1, epsilon2;
        force.getExceptionParameters(group1, particle1, particle2, chargeProd1, sigma1, epsilon1);
        force.getExceptionParameters(group2, particle1, particle2, chargeProd2, sigma2, epsilon2);
        return (chargeProd1 == chargeProd2 && sigma1 == sigma2 && epsilon1 == epsilon2);
    }
private:
    const NonbondedForce& force;
};

1446
1447
class OpenCLCalcNonbondedForceKernel::PmeIO : public CalcPmeReciprocalForceKernel::IO {
public:
peastman's avatar
peastman committed
1448
1449
1450
    PmeIO(OpenCLContext& cl, cl::Kernel addForcesKernel) : cl(cl), addForcesKernel(addForcesKernel) {
        forceTemp.initialize<mm_float4>(cl, cl.getNumAtoms(), "PmeForce");
        addForcesKernel.setArg<cl::Buffer>(0, forceTemp.getDeviceBuffer());
1451
1452
1453
1454
1455
1456
    }
    float* getPosq() {
        cl.getPosq().download(posq);
        return (float*) &posq[0];
    }
    void setForce(float* force) {
peastman's avatar
peastman committed
1457
        forceTemp.upload(force);
1458
1459
1460
1461
1462
1463
        addForcesKernel.setArg<cl::Buffer>(1, cl.getForce().getDeviceBuffer());
        cl.executeKernel(addForcesKernel, cl.getNumAtoms());
    }
private:
    OpenCLContext& cl;
    vector<mm_float4> posq;
peastman's avatar
peastman committed
1464
    OpenCLArray forceTemp;
1465
1466
1467
1468
1469
1470
1471
1472
    cl::Kernel addForcesKernel;
};

class OpenCLCalcNonbondedForceKernel::PmePreComputation : public OpenCLContext::ForcePreComputation {
public:
    PmePreComputation(OpenCLContext& cl, Kernel& pme, CalcPmeReciprocalForceKernel::IO& io) : cl(cl), pme(pme), io(io) {
    }
    void computeForceAndEnergy(bool includeForces, bool includeEnergy, int groups) {
peastman's avatar
peastman committed
1473
1474
        Vec3 boxVectors[3] = {Vec3(cl.getPeriodicBoxSize().x, 0, 0), Vec3(0, cl.getPeriodicBoxSize().y, 0), Vec3(0, 0, cl.getPeriodicBoxSize().z)};
        pme.getAs<CalcPmeReciprocalForceKernel>().beginComputation(io, boxVectors, includeEnergy);
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
    }
private:
    OpenCLContext& cl;
    Kernel pme;
    CalcPmeReciprocalForceKernel::IO& io;
};

class OpenCLCalcNonbondedForceKernel::PmePostComputation : public OpenCLContext::ForcePostComputation {
public:
    PmePostComputation(Kernel& pme, CalcPmeReciprocalForceKernel::IO& io) : pme(pme), io(io) {
    }
    double computeForceAndEnergy(bool includeForces, bool includeEnergy, int groups) {
        return pme.getAs<CalcPmeReciprocalForceKernel>().finishComputation(io);
    }
private:
    Kernel pme;
    CalcPmeReciprocalForceKernel::IO& io;
};

1494
1495
class OpenCLCalcNonbondedForceKernel::SyncQueuePreComputation : public OpenCLContext::ForcePreComputation {
public:
1496
    SyncQueuePreComputation(OpenCLContext& cl, cl::CommandQueue queue, int forceGroup) : cl(cl), queue(queue), forceGroup(forceGroup) {
1497
1498
    }
    void computeForceAndEnergy(bool includeForces, bool includeEnergy, int groups) {
peastman's avatar
Bug fix  
peastman committed
1499
        if ((groups&(1<<forceGroup)) != 0) {
1500
            vector<cl::Event> events(1);
peastman's avatar
Bug fix  
peastman committed
1501
1502
1503
            cl.getQueue().enqueueMarker(&events[0]);
            queue.enqueueWaitForEvents(events);
        }
1504
1505
1506
1507
    }
private:
    OpenCLContext& cl;
    cl::CommandQueue queue;
peastman's avatar
Bug fix  
peastman committed
1508
    int forceGroup;
1509
1510
1511
1512
};

class OpenCLCalcNonbondedForceKernel::SyncQueuePostComputation : public OpenCLContext::ForcePostComputation {
public:
1513
1514
1515
1516
1517
1518
1519
1520
    SyncQueuePostComputation(OpenCLContext& cl, cl::Event& event, OpenCLArray& pmeEnergyBuffer, int forceGroup) : cl(cl), event(event),
            pmeEnergyBuffer(pmeEnergyBuffer), forceGroup(forceGroup) {
    }
    void setKernel(cl::Kernel kernel) {
        addEnergyKernel = kernel;
        addEnergyKernel.setArg<cl::Buffer>(0, pmeEnergyBuffer.getDeviceBuffer());
        addEnergyKernel.setArg<cl::Buffer>(1, cl.getEnergyBuffer().getDeviceBuffer());
        addEnergyKernel.setArg<cl_int>(2, pmeEnergyBuffer.getSize());
1521
1522
    }
    double computeForceAndEnergy(bool includeForces, bool includeEnergy, int groups) {
peastman's avatar
Bug fix  
peastman committed
1523
        if ((groups&(1<<forceGroup)) != 0) {
1524
            vector<cl::Event> events(1);
peastman's avatar
Bug fix  
peastman committed
1525
            events[0] = event;
1526
            event = cl::Event();
peastman's avatar
Bug fix  
peastman committed
1527
            cl.getQueue().enqueueWaitForEvents(events);
1528
1529
            if (includeEnergy)
                cl.executeKernel(addEnergyKernel, pmeEnergyBuffer.getSize());
peastman's avatar
Bug fix  
peastman committed
1530
        }
1531
1532
1533
1534
1535
        return 0.0;
    }
private:
    OpenCLContext& cl;
    cl::Event& event;
1536
1537
    cl::Kernel addEnergyKernel;
    OpenCLArray& pmeEnergyBuffer;
peastman's avatar
Bug fix  
peastman committed
1538
    int forceGroup;
1539
1540
};

1541
OpenCLCalcNonbondedForceKernel::~OpenCLCalcNonbondedForceKernel() {
1542
1543
1544
1545
    if (sort != NULL)
        delete sort;
    if (fft != NULL)
        delete fft;
1546
1547
    if (dispersionFft != NULL)
        delete dispersionFft;
1548
1549
    if (pmeio != NULL)
        delete pmeio;
1550
1551
1552
}

void OpenCLCalcNonbondedForceKernel::initialize(const System& system, const NonbondedForce& force) {
1553
1554
1555
1556
    int forceIndex;
    for (forceIndex = 0; forceIndex < system.getNumForces() && &system.getForce(forceIndex) != &force; ++forceIndex)
        ;
    string prefix = "nonbonded"+cl.intToString(forceIndex)+"_";
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573

    // Identify which exceptions are 1-4 interactions.

    vector<pair<int, int> > exclusions;
    vector<int> exceptions;
    for (int i = 0; i < force.getNumExceptions(); i++) {
        int particle1, particle2;
        double chargeProd, sigma, epsilon;
        force.getExceptionParameters(i, particle1, particle2, chargeProd, sigma, epsilon);
        exclusions.push_back(pair<int, int>(particle1, particle2));
        if (chargeProd != 0.0 || epsilon != 0.0)
            exceptions.push_back(i);
    }

    // Initialize nonbonded interactions.

    int numParticles = force.getNumParticles();
1574
    vector<mm_float4> baseParticleParamVec(cl.getPaddedNumAtoms(), mm_float4(0, 0, 0, 0));
1575
    vector<vector<int> > exclusionList(numParticles);
1576
1577
    hasCoulomb = false;
    hasLJ = false;
1578
1579
1580
    for (int i = 0; i < numParticles; i++) {
        double charge, sigma, epsilon;
        force.getParticleParameters(i, charge, sigma, epsilon);
1581
        baseParticleParamVec[i] = mm_float4(charge, sigma, epsilon, 0);
1582
        exclusionList[i].push_back(i);
1583
1584
1585
1586
        if (charge != 0.0)
            hasCoulomb = true;
        if (epsilon != 0.0)
            hasLJ = true;
1587
    }
peastman's avatar
peastman committed
1588
1589
1590
    for (auto exclusion : exclusions) {
        exclusionList[exclusion.first].push_back(exclusion.second);
        exclusionList[exclusion.second].push_back(exclusion.first);
1591
    }
1592
1593
1594
    nonbondedMethod = CalcNonbondedForceKernel::NonbondedMethod(force.getNonbondedMethod());
    bool useCutoff = (nonbondedMethod != NoCutoff);
    bool usePeriodic = (nonbondedMethod != NoCutoff && nonbondedMethod != CutoffNonPeriodic);
1595
    doLJPME = (nonbondedMethod == LJPME && hasLJ);
Peter Eastman's avatar
Peter Eastman committed
1596
1597
    if (hasCoulomb)
        usePosqCharges = cl.requestPosqCharges();
1598
    map<string, string> defines;
1599
1600
    defines["HAS_COULOMB"] = (hasCoulomb ? "1" : "0");
    defines["HAS_LENNARD_JONES"] = (hasLJ ? "1" : "0");
1601
    defines["USE_LJ_SWITCH"] = (useCutoff && force.getUseSwitchingFunction() ? "1" : "0");
1602
    if (useCutoff) {
1603
1604
        // Compute the reaction field constants.

1605
1606
        double reactionFieldK = pow(force.getCutoffDistance(), -3.0)*(force.getReactionFieldDielectric()-1.0)/(2.0*force.getReactionFieldDielectric()+1.0);
        double reactionFieldC = (1.0 / force.getCutoffDistance())*(3.0*force.getReactionFieldDielectric())/(2.0*force.getReactionFieldDielectric()+1.0);
1607
1608
        defines["REACTION_FIELD_K"] = cl.doubleToString(reactionFieldK);
        defines["REACTION_FIELD_C"] = cl.doubleToString(reactionFieldC);
1609
1610
1611
1612
1613
1614
1615
1616
1617
        
        // Compute the switching coefficients.
        
        if (force.getUseSwitchingFunction()) {
            defines["LJ_SWITCH_CUTOFF"] = cl.doubleToString(force.getSwitchingDistance());
            defines["LJ_SWITCH_C3"] = cl.doubleToString(10/pow(force.getSwitchingDistance()-force.getCutoffDistance(), 3.0));
            defines["LJ_SWITCH_C4"] = cl.doubleToString(15/pow(force.getSwitchingDistance()-force.getCutoffDistance(), 4.0));
            defines["LJ_SWITCH_C5"] = cl.doubleToString(6/pow(force.getSwitchingDistance()-force.getCutoffDistance(), 5.0));
        }
1618
    }
1619
    if (force.getUseDispersionCorrection() && cl.getContextIndex() == 0 && !doLJPME)
1620
1621
1622
        dispersionCoefficient = NonbondedForceImpl::calcDispersionCorrection(system, force);
    else
        dispersionCoefficient = 0.0;
1623
    alpha = 0;
1624
    ewaldSelfEnergy = 0.0;
1625
    map<string, string> paramsDefines;
Peter Eastman's avatar
Bug fix  
Peter Eastman committed
1626
1627
    hasOffsets = (force.getNumParticleParameterOffsets() > 0 || force.getNumExceptionParameterOffsets() > 0);
    if (hasOffsets)
1628
        paramsDefines["HAS_OFFSETS"] = "1";
1629
    if (nonbondedMethod == Ewald) {
1630
1631
1632
1633
        // Compute the Ewald parameters.

        int kmaxx, kmaxy, kmaxz;
        NonbondedForceImpl::calcEwaldParameters(system, force, alpha, kmaxx, kmaxy, kmaxz);
1634
1635
        defines["EWALD_ALPHA"] = cl.doubleToString(alpha);
        defines["TWO_OVER_SQRT_PI"] = cl.doubleToString(2.0/sqrt(M_PI));
1636
        defines["USE_EWALD"] = "1";
1637
        if (cl.getContextIndex() == 0) {
1638
1639
            paramsDefines["INCLUDE_EWALD"] = "1";
            paramsDefines["EWALD_SELF_ENERGY_SCALE"] = cl.doubleToString(ONE_4PI_EPS0*alpha/sqrt(M_PI));
Peter Eastman's avatar
Peter Eastman committed
1640
1641
            for (int i = 0; i < numParticles; i++)
                ewaldSelfEnergy += baseParticleParamVec[i].x*baseParticleParamVec[i].x*ONE_4PI_EPS0*alpha/sqrt(M_PI);
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654

            // Create the reciprocal space kernels.

            map<string, string> replacements;
            replacements["NUM_ATOMS"] = cl.intToString(numParticles);
            replacements["KMAX_X"] = cl.intToString(kmaxx);
            replacements["KMAX_Y"] = cl.intToString(kmaxy);
            replacements["KMAX_Z"] = cl.intToString(kmaxz);
            replacements["EXP_COEFFICIENT"] = cl.doubleToString(-1.0/(4.0*alpha*alpha));
            cl::Program program = cl.createProgram(OpenCLKernelSources::ewald, replacements);
            ewaldSumsKernel = cl::Kernel(program, "calculateEwaldCosSinSums");
            ewaldForcesKernel = cl::Kernel(program, "calculateEwaldForces");
            int elementSize = (cl.getUseDoublePrecision() ? sizeof(mm_double2) : sizeof(mm_float2));
peastman's avatar
peastman committed
1655
            cosSinSums.initialize(cl, (2*kmaxx-1)*(2*kmaxy-1)*(2*kmaxz-1), elementSize, "cosSinSums");
1656
1657
        }
    }
peastman's avatar
peastman committed
1658
    else if (((nonbondedMethod == PME || nonbondedMethod == LJPME) && hasCoulomb) || doLJPME) {
1659
1660
        // Compute the PME parameters.

1661
        NonbondedForceImpl::calcPMEParameters(system, force, alpha, gridSizeX, gridSizeY, gridSizeZ, false);
1662
1663
1664
        gridSizeX = OpenCLFFT3D::findLegalDimension(gridSizeX);
        gridSizeY = OpenCLFFT3D::findLegalDimension(gridSizeY);
        gridSizeZ = OpenCLFFT3D::findLegalDimension(gridSizeZ);
1665
1666
1667
1668
1669
1670
1671
        if (doLJPME) {
            NonbondedForceImpl::calcPMEParameters(system, force, dispersionAlpha, dispersionGridSizeX,
                                                  dispersionGridSizeY, dispersionGridSizeZ, true);
            dispersionGridSizeX = OpenCLFFT3D::findLegalDimension(dispersionGridSizeX);
            dispersionGridSizeY = OpenCLFFT3D::findLegalDimension(dispersionGridSizeY);
            dispersionGridSizeZ = OpenCLFFT3D::findLegalDimension(dispersionGridSizeZ);
        }
1672
1673
        defines["EWALD_ALPHA"] = cl.doubleToString(alpha);
        defines["TWO_OVER_SQRT_PI"] = cl.doubleToString(2.0/sqrt(M_PI));
1674
        defines["USE_EWALD"] = "1";
1675
1676
1677
        defines["DO_LJPME"] = doLJPME ? "1" : "0";
        if (doLJPME)
            defines["EWALD_DISPERSION_ALPHA"] = cl.doubleToString(dispersionAlpha);
1678
        if (cl.getContextIndex() == 0) {
1679
1680
            paramsDefines["INCLUDE_EWALD"] = "1";
            paramsDefines["EWALD_SELF_ENERGY_SCALE"] = cl.doubleToString(ONE_4PI_EPS0*alpha/sqrt(M_PI));
Peter Eastman's avatar
Peter Eastman committed
1681
            for (int i = 0; i < numParticles; i++)
1682
                ewaldSelfEnergy -= baseParticleParamVec[i].x*baseParticleParamVec[i].x*ONE_4PI_EPS0*alpha/sqrt(M_PI);
1683
1684
1685
            if (doLJPME) {
                paramsDefines["INCLUDE_LJPME"] = "1";
                paramsDefines["LJPME_SELF_ENERGY_SCALE"] = cl.doubleToString(pow(dispersionAlpha, 6)/3.0);
Peter Eastman's avatar
Peter Eastman committed
1686
1687
                for (int i = 0; i < numParticles; i++)
                    ewaldSelfEnergy += baseParticleParamVec[i].z*pow(baseParticleParamVec[i].y*dispersionAlpha, 6)/3.0;
1688
            }
1689
1690
1691
1692
1693
1694
1695
            pmeDefines["PME_ORDER"] = cl.intToString(PmeOrder);
            pmeDefines["NUM_ATOMS"] = cl.intToString(numParticles);
            pmeDefines["RECIP_EXP_FACTOR"] = cl.doubleToString(M_PI*M_PI/(alpha*alpha));
            pmeDefines["GRID_SIZE_X"] = cl.intToString(gridSizeX);
            pmeDefines["GRID_SIZE_Y"] = cl.intToString(gridSizeY);
            pmeDefines["GRID_SIZE_Z"] = cl.intToString(gridSizeZ);
            pmeDefines["EPSILON_FACTOR"] = cl.doubleToString(sqrt(ONE_4PI_EPS0));
1696
            pmeDefines["M_PI"] = cl.doubleToString(M_PI);
1697
1698
1699
            bool deviceIsCpu = (cl.getDevice().getInfo<CL_DEVICE_TYPE>() == CL_DEVICE_TYPE_CPU);
            if (deviceIsCpu)
                pmeDefines["DEVICE_IS_CPU"] = "1";
1700
            if (cl.getPlatformData().useCpuPme && !doLJPME && usePosqCharges) {
1701
1702
1703
1704
                // Create the CPU PME kernel.

                try {
                    cpuPme = getPlatform().createKernel(CalcPmeReciprocalForceKernel::Name(), *cl.getPlatformData().context);
1705
                    cpuPme.getAs<CalcPmeReciprocalForceKernel>().initialize(gridSizeX, gridSizeY, gridSizeZ, numParticles, alpha, false);
1706
1707
1708
1709
1710
                    cl::Program program = cl.createProgram(OpenCLKernelSources::pme, pmeDefines);
                    cl::Kernel addForcesKernel = cl::Kernel(program, "addForces");
                    pmeio = new PmeIO(cl, addForcesKernel);
                    cl.addPreComputation(new PmePreComputation(cl, cpuPme, *pmeio));
                    cl.addPostComputation(new PmePostComputation(cpuPme, *pmeio));
1711
                }
1712
1713
                catch (OpenMMException& ex) {
                    // The CPU PME plugin isn't available.
1714
                }
1715
1716
1717
1718
            }
            if (pmeio == NULL) {
                // Create required data structures.

1719
1720
1721
1722
1723
1724
1725
1726
1727
                if (doLJPME) {
                    double invRCut6 = pow(force.getCutoffDistance(), -6);
                    double dalphaR = dispersionAlpha * force.getCutoffDistance();
                    double dar2 = dalphaR*dalphaR;
                    double dar4 = dar2*dar2;
                    double multShift6 = -invRCut6*(1.0 - exp(-dar2) * (1.0 + dar2 + 0.5*dar4));
                    defines["INVCUT6"] = cl.doubleToString(invRCut6);
                    defines["MULTSHIFT6"] = cl.doubleToString(multShift6);
                }
1728
                int elementSize = (cl.getUseDoublePrecision() ? sizeof(double) : sizeof(float));
1729
1730
1731
                int gridElements = gridSizeX*gridSizeY*gridSizeZ;
                if (doLJPME)
                    gridElements = max(gridElements, dispersionGridSizeX*dispersionGridSizeY*dispersionGridSizeZ);
peastman's avatar
peastman committed
1732
1733
                pmeGrid.initialize(cl, gridElements, 2*elementSize, "pmeGrid");
                pmeGrid2.initialize(cl, gridElements, 2*elementSize, "pmeGrid2");
peastman's avatar
peastman committed
1734
                if (cl.getSupports64BitGlobalAtomics())
peastman's avatar
peastman committed
1735
                    cl.addAutoclearBuffer(pmeGrid2);
peastman's avatar
peastman committed
1736
                else
peastman's avatar
peastman committed
1737
1738
1739
1740
                    cl.addAutoclearBuffer(pmeGrid);
                pmeBsplineModuliX.initialize(cl, gridSizeX, elementSize, "pmeBsplineModuliX");
                pmeBsplineModuliY.initialize(cl, gridSizeY, elementSize, "pmeBsplineModuliY");
                pmeBsplineModuliZ.initialize(cl, gridSizeZ, elementSize, "pmeBsplineModuliZ");
1741
                if (doLJPME) {
peastman's avatar
peastman committed
1742
1743
1744
                    pmeDispersionBsplineModuliX.initialize(cl, dispersionGridSizeX, elementSize, "pmeDispersionBsplineModuliX");
                    pmeDispersionBsplineModuliY.initialize(cl, dispersionGridSizeY, elementSize, "pmeDispersionBsplineModuliY");
                    pmeDispersionBsplineModuliZ.initialize(cl, dispersionGridSizeZ, elementSize, "pmeDispersionBsplineModuliZ");
1745
                }
peastman's avatar
peastman committed
1746
1747
1748
                pmeBsplineTheta.initialize(cl, PmeOrder*numParticles, 4*elementSize, "pmeBsplineTheta");
                pmeAtomRange.initialize<cl_int>(cl, gridSizeX*gridSizeY*gridSizeZ+1, "pmeAtomRange");
                pmeAtomGridIndex.initialize<mm_int2>(cl, numParticles, "pmeAtomGridIndex");
1749
                int energyElementSize = (cl.getUseDoublePrecision() || cl.getUseMixedPrecision() ? sizeof(double) : sizeof(float));
peastman's avatar
peastman committed
1750
1751
                pmeEnergyBuffer.initialize(cl, cl.getNumThreadBlocks()*OpenCLContext::ThreadBlockSize, energyElementSize, "pmeEnergyBuffer");
                cl.clearBuffer(pmeEnergyBuffer);
1752
                sort = new OpenCLSort(cl, new SortTrait(), cl.getNumAtoms());
1753
                fft = new OpenCLFFT3D(cl, gridSizeX, gridSizeY, gridSizeZ, true);
1754
1755
                if (doLJPME)
                    dispersionFft = new OpenCLFFT3D(cl, dispersionGridSizeX, dispersionGridSizeY, dispersionGridSizeZ, true);
1756
                string vendor = cl.getDevice().getInfo<CL_DEVICE_VENDOR>();
Peter Eastman's avatar
Peter Eastman committed
1757
1758
1759
                bool isNvidia = (vendor.size() >= 6 && vendor.substr(0, 6) == "NVIDIA");
                if (isNvidia)
                    pmeDefines["USE_ALTERNATE_MEMORY_ACCESS_PATTERN"] = "1";
1760
                usePmeQueue = (!cl.getPlatformData().disablePmeStream && isNvidia);
1761
                if (usePmeQueue) {
peastman's avatar
peastman committed
1762
                    pmeDefines["USE_PME_STREAM"] = "1";
1763
1764
1765
1766
1767
                    pmeQueue = cl::CommandQueue(cl.getContext(), cl.getDevice());
                    int recipForceGroup = force.getReciprocalSpaceForceGroup();
                    if (recipForceGroup < 0)
                        recipForceGroup = force.getForceGroup();
                    cl.addPreComputation(new SyncQueuePreComputation(cl, pmeQueue, recipForceGroup));
peastman's avatar
peastman committed
1768
                    cl.addPostComputation(syncQueue = new SyncQueuePostComputation(cl, pmeSyncEvent, pmeEnergyBuffer, recipForceGroup));
1769
                }
1770
1771
1772

                // Initialize the b-spline moduli.

1773
1774
1775
1776
1777
1778
1779
                for (int grid = 0; grid < 2; grid++) {
                    int xsize, ysize, zsize;
                    OpenCLArray *xmoduli, *ymoduli, *zmoduli;
                    if (grid == 0) {
                        xsize = gridSizeX;
                        ysize = gridSizeY;
                        zsize = gridSizeZ;
peastman's avatar
peastman committed
1780
1781
1782
                        xmoduli = &pmeBsplineModuliX;
                        ymoduli = &pmeBsplineModuliY;
                        zmoduli = &pmeBsplineModuliZ;
1783
                    }
1784
1785
1786
1787
1788
1789
                    else {
                        if (!doLJPME)
                            continue;
                        xsize = dispersionGridSizeX;
                        ysize = dispersionGridSizeY;
                        zsize = dispersionGridSizeZ;
peastman's avatar
peastman committed
1790
1791
1792
                        xmoduli = &pmeDispersionBsplineModuliX;
                        ymoduli = &pmeDispersionBsplineModuliY;
                        zmoduli = &pmeDispersionBsplineModuliZ;
1793
                    }
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
                    int maxSize = max(max(xsize, ysize), zsize);
                    vector<double> data(PmeOrder);
                    vector<double> ddata(PmeOrder);
                    vector<double> bsplines_data(maxSize);
                    data[PmeOrder-1] = 0.0;
                    data[1] = 0.0;
                    data[0] = 1.0;
                    for (int i = 3; i < PmeOrder; i++) {
                        double div = 1.0/(i-1.0);
                        data[i-1] = 0.0;
                        for (int j = 1; j < (i-1); j++)
                            data[i-j-1] = div*(j*data[i-j-2]+(i-j)*data[i-j-1]);
                        data[0] = div*data[0];
1807
                    }
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838

                    // Differentiate.

                    ddata[0] = -data[0];
                    for (int i = 1; i < PmeOrder; i++)
                        ddata[i] = data[i-1]-data[i];
                    double div = 1.0/(PmeOrder-1);
                    data[PmeOrder-1] = 0.0;
                    for (int i = 1; i < (PmeOrder-1); i++)
                        data[PmeOrder-i-1] = div*(i*data[PmeOrder-i-2]+(PmeOrder-i)*data[PmeOrder-i-1]);
                    data[0] = div*data[0];
                    for (int i = 0; i < maxSize; i++)
                        bsplines_data[i] = 0.0;
                    for (int i = 1; i <= PmeOrder; i++)
                        bsplines_data[i] = data[i-1];

                    // Evaluate the actual bspline moduli for X/Y/Z.

                    for(int dim = 0; dim < 3; dim++) {
                        int ndata = (dim == 0 ? xsize : dim == 1 ? ysize : zsize);
                        vector<cl_double> moduli(ndata);
                        for (int i = 0; i < ndata; i++) {
                            double sc = 0.0;
                            double ss = 0.0;
                            for (int j = 0; j < ndata; j++) {
                                double arg = (2.0*M_PI*i*j)/ndata;
                                sc += bsplines_data[j]*cos(arg);
                                ss += bsplines_data[j]*sin(arg);
                            }
                            moduli[i] = (float) (sc*sc+ss*ss);
                        }
1839
                        for (int i = 0; i < ndata; i++)
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
                        {
                            if (moduli[i] < 1.0e-7)
                                moduli[i] = (moduli[i-1]+moduli[i+1])*0.5f;
                        }
                        if (cl.getUseDoublePrecision()) {
                            if (dim == 0)
                                xmoduli->upload(moduli);
                            else if (dim == 1)
                                ymoduli->upload(moduli);
                            else
                                zmoduli->upload(moduli);
                        }
                        else {
                            vector<float> modulif(ndata);
                            for (int i = 0; i < ndata; i++)
                                modulif[i] = (float) moduli[i];
                            if (dim == 0)
                                xmoduli->upload(modulif);
                            else if (dim == 1)
                                ymoduli->upload(modulif);
                            else
                                zmoduli->upload(modulif);
                        }
1863
                    }
1864
                }
1865
            }
1866
1867
        }
    }
1868
1869
1870

    // Add the interaction to the default nonbonded kernel.
    
1871
    string source = cl.replaceStrings(OpenCLKernelSources::coulombLennardJones, defines);
1872
    charges.initialize(cl, cl.getPaddedNumAtoms(), cl.getUseDoublePrecision() ? sizeof(double) : sizeof(float), "charges");
1873
1874
    baseParticleParams.initialize<mm_float4>(cl, cl.getPaddedNumAtoms(), "baseParticleParams");
    baseParticleParams.upload(baseParticleParamVec);
peastman's avatar
peastman committed
1875
1876
1877
1878
    map<string, string> replacements;
    if (usePosqCharges) {
        replacements["CHARGE1"] = "posq1.w";
        replacements["CHARGE2"] = "posq2.w";
1879
        paramsDefines["USE_POSQ_CHARGES"] = "1";
peastman's avatar
peastman committed
1880
1881
1882
1883
    }
    else {
        replacements["CHARGE1"] = prefix+"charge1";
        replacements["CHARGE2"] = prefix+"charge2";
1884
    }
peastman's avatar
peastman committed
1885
1886
    if (hasCoulomb)
        cl.getNonbondedUtilities().addParameter(OpenCLNonbondedUtilities::ParameterInfo(prefix+"charge", "real", 1, charges.getElementSize(), charges.getDeviceBuffer()));
1887
    sigmaEpsilon.initialize<mm_float2>(cl, cl.getPaddedNumAtoms(), "sigmaEpsilon");
1888
1889
1890
1891
1892
    if (hasLJ) {
        replacements["SIGMA_EPSILON1"] = prefix+"sigmaEpsilon1";
        replacements["SIGMA_EPSILON2"] = prefix+"sigmaEpsilon2";
        cl.getNonbondedUtilities().addParameter(OpenCLNonbondedUtilities::ParameterInfo(prefix+"sigmaEpsilon", "float", 2, sizeof(cl_float2), sigmaEpsilon.getDeviceBuffer()));
    }
peastman's avatar
peastman committed
1893
    source = cl.replaceStrings(source, replacements);
1894
    cl.getNonbondedUtilities().addInteraction(useCutoff, usePeriodic, true, force.getCutoffDistance(), exclusionList, source, force.getForceGroup());
1895

1896
    // Initialize the exceptions.
1897

1898
1899
1900
1901
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*exceptions.size()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*exceptions.size()/numContexts;
    int numExceptions = endIndex-startIndex;
1902
    if (numExceptions > 0) {
1903
        paramsDefines["HAS_EXCEPTIONS"] = "1";
1904
        exceptionAtoms.resize(numExceptions);
Peter Eastman's avatar
Peter Eastman committed
1905
        vector<vector<int> > atoms(numExceptions, vector<int>(2));
peastman's avatar
peastman committed
1906
        exceptionParams.initialize<mm_float4>(cl, numExceptions, "exceptionParams");
1907
1908
        baseExceptionParams.initialize<mm_float4>(cl, numExceptions, "baseExceptionParams");
        vector<mm_float4> baseExceptionParamsVec(numExceptions);
1909
        for (int i = 0; i < numExceptions; i++) {
1910
            double chargeProd, sigma, epsilon;
Peter Eastman's avatar
Peter Eastman committed
1911
            force.getExceptionParameters(exceptions[startIndex+i], atoms[i][0], atoms[i][1], chargeProd, sigma, epsilon);
1912
            baseExceptionParamsVec[i] = mm_float4(chargeProd, sigma, epsilon, 0);
1913
            exceptionAtoms[i] = make_pair(atoms[i][0], atoms[i][1]);
1914
        }
1915
        baseExceptionParams.upload(baseExceptionParamsVec);
Peter Eastman's avatar
Peter Eastman committed
1916
        map<string, string> replacements;
peastman's avatar
peastman committed
1917
        replacements["PARAMS"] = cl.getBondedUtilities().addArgument(exceptionParams.getDeviceBuffer(), "float4");
1918
        cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::nonbondedExceptions, replacements), force.getForceGroup());
Peter Eastman's avatar
Peter Eastman committed
1919
    }
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
    
    // Initialize parameter offsets.

    vector<vector<mm_float4> > particleOffsetVec(force.getNumParticles());
    vector<vector<mm_float4> > exceptionOffsetVec(force.getNumExceptions());
    for (int i = 0; i < force.getNumParticleParameterOffsets(); i++) {
        string param;
        int particle;
        double charge, sigma, epsilon;
        force.getParticleParameterOffset(i, param, particle, charge, sigma, epsilon);
        auto paramPos = find(paramNames.begin(), paramNames.end(), param);
        int paramIndex;
        if (paramPos == paramNames.end()) {
            paramIndex = paramNames.size();
            paramNames.push_back(param);
        }
        else
            paramIndex = paramPos-paramNames.begin();
        particleOffsetVec[particle].push_back(mm_float4(charge, sigma, epsilon, paramIndex));
    }
    for (int i = 0; i < force.getNumExceptionParameterOffsets(); i++) {
        string param;
        int exception;
        double charge, sigma, epsilon;
        force.getExceptionParameterOffset(i, param, exception, charge, sigma, epsilon);
        auto paramPos = find(paramNames.begin(), paramNames.end(), param);
        int paramIndex;
        if (paramPos == paramNames.end()) {
            paramIndex = paramNames.size();
            paramNames.push_back(param);
        }
        else
            paramIndex = paramPos-paramNames.begin();
        exceptionOffsetVec[exception].push_back(mm_float4(charge, sigma, epsilon, paramIndex));
    }
    paramValues.resize(paramNames.size(), 0.0);
    particleParamOffsets.initialize<mm_float4>(cl, max(force.getNumParticleParameterOffsets(), 1), "particleParamOffsets");
    exceptionParamOffsets.initialize<mm_float4>(cl, max(force.getNumExceptionParameterOffsets(), 1), "exceptionParamOffsets");
Peter Eastman's avatar
Bug fix  
Peter Eastman committed
1958
    particleOffsetIndices.initialize<cl_int>(cl, cl.getPaddedNumAtoms()+1, "particleOffsetIndices");
1959
1960
1961
1962
1963
1964
1965
1966
    exceptionOffsetIndices.initialize<cl_int>(cl, force.getNumExceptions()+1, "exceptionOffsetIndices");
    vector<cl_int> particleOffsetIndicesVec, exceptionOffsetIndicesVec;
    vector<mm_float4> p, e;
    for (int i = 0; i < particleOffsetVec.size(); i++) {
        particleOffsetIndicesVec.push_back(p.size());
        for (int j = 0; j < particleOffsetVec[i].size(); j++)
            p.push_back(particleOffsetVec[i][j]);
    }
Peter Eastman's avatar
Bug fix  
Peter Eastman committed
1967
1968
    while (particleOffsetIndicesVec.size() < particleOffsetIndices.getSize())
        particleOffsetIndicesVec.push_back(p.size());
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
    for (int i = 0; i < exceptionOffsetVec.size(); i++) {
        exceptionOffsetIndicesVec.push_back(e.size());
        for (int j = 0; j < exceptionOffsetVec[i].size(); j++)
            e.push_back(exceptionOffsetVec[i][j]);
    }
    exceptionOffsetIndicesVec.push_back(e.size());
    if (force.getNumParticleParameterOffsets() > 0) {
        particleParamOffsets.upload(p);
        particleOffsetIndices.upload(particleOffsetIndicesVec);
    }
    if (force.getNumExceptionParameterOffsets() > 0) {
        exceptionParamOffsets.upload(e);
        exceptionOffsetIndices.upload(exceptionOffsetIndicesVec);
    }
    globalParams.initialize(cl, max((int) paramValues.size(), 1), cl.getUseDoublePrecision() ? sizeof(double) : sizeof(float), "globalParams");
Peter Eastman's avatar
Peter Eastman committed
1984
    recomputeParams = true;
1985
1986
1987
1988
1989
    
    // Initialize the kernel for updating parameters.
    
    cl::Program program = cl.createProgram(OpenCLKernelSources::nonbondedParameters, paramsDefines);
    computeParamsKernel = cl::Kernel(program, "computeParameters");
1990
1991
    info = new ForceInfo(cl.getNonbondedUtilities().getNumForceBuffers(), force);
    cl.addForce(info);
1992
1993
}

1994
double OpenCLCalcNonbondedForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy, bool includeDirect, bool includeReciprocal) {
1995
    bool deviceIsCpu = (cl.getDevice().getInfo<CL_DEVICE_TYPE>() == CL_DEVICE_TYPE_CPU);
1996
1997
    if (!hasInitializedKernel) {
        hasInitializedKernel = true;
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
        computeParamsKernel.setArg<cl::Buffer>(0, cl.getEnergyBuffer().getDeviceBuffer());
        computeParamsKernel.setArg<cl::Buffer>(2, globalParams.getDeviceBuffer());
        computeParamsKernel.setArg<cl_int>(3, cl.getPaddedNumAtoms());
        computeParamsKernel.setArg<cl::Buffer>(4, baseParticleParams.getDeviceBuffer());
        computeParamsKernel.setArg<cl::Buffer>(5, cl.getPosq().getDeviceBuffer());
        computeParamsKernel.setArg<cl::Buffer>(6, charges.getDeviceBuffer());
        computeParamsKernel.setArg<cl::Buffer>(7, sigmaEpsilon.getDeviceBuffer());
        computeParamsKernel.setArg<cl::Buffer>(8, particleParamOffsets.getDeviceBuffer());
        computeParamsKernel.setArg<cl::Buffer>(9, particleOffsetIndices.getDeviceBuffer());
        if (exceptionParams.isInitialized()) {
            computeParamsKernel.setArg<cl_int>(10, exceptionParams.getSize());
            computeParamsKernel.setArg<cl::Buffer>(11, baseExceptionParams.getDeviceBuffer());
            computeParamsKernel.setArg<cl::Buffer>(12, exceptionParams.getDeviceBuffer());
            computeParamsKernel.setArg<cl::Buffer>(13, exceptionParamOffsets.getDeviceBuffer());
            computeParamsKernel.setArg<cl::Buffer>(14, exceptionOffsetIndices.getDeviceBuffer());
        }
peastman's avatar
peastman committed
2014
        if (cosSinSums.isInitialized()) {
2015
2016
            ewaldSumsKernel.setArg<cl::Buffer>(0, cl.getEnergyBuffer().getDeviceBuffer());
            ewaldSumsKernel.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2017
            ewaldSumsKernel.setArg<cl::Buffer>(2, cosSinSums.getDeviceBuffer());
2018
2019
            ewaldForcesKernel.setArg<cl::Buffer>(0, cl.getForceBuffers().getDeviceBuffer());
            ewaldForcesKernel.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2020
            ewaldForcesKernel.setArg<cl::Buffer>(2, cosSinSums.getDeviceBuffer());
2021
        }
peastman's avatar
peastman committed
2022
        if (pmeGrid.isInitialized()) {
2023
2024
            // Create kernels for Coulomb PME.
            
2025
2026
2027
            map<string, string> replacements;
            replacements["CHARGE"] = (usePosqCharges ? "pos.w" : "charges[atom]");
            cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::pme, replacements), pmeDefines);
2028
            pmeUpdateBsplinesKernel = cl::Kernel(program, "updateBsplines");
2029
            pmeAtomRangeKernel = cl::Kernel(program, "findAtomRangeForGrid");
2030
            pmeZIndexKernel = cl::Kernel(program, "recordZIndex");
2031
2032
            pmeSpreadChargeKernel = cl::Kernel(program, "gridSpreadCharge");
            pmeConvolutionKernel = cl::Kernel(program, "reciprocalConvolution");
2033
            pmeEvalEnergyKernel = cl::Kernel(program, "gridEvaluateEnergy");
2034
            pmeInterpolateForceKernel = cl::Kernel(program, "gridInterpolateForce");
2035
            int elementSize = (cl.getUseDoublePrecision() ? sizeof(mm_double4) : sizeof(mm_float4));
2036
            pmeUpdateBsplinesKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2037
            pmeUpdateBsplinesKernel.setArg<cl::Buffer>(1, pmeBsplineTheta.getDeviceBuffer());
2038
            pmeUpdateBsplinesKernel.setArg(2, OpenCLContext::ThreadBlockSize*PmeOrder*elementSize, NULL);
peastman's avatar
peastman committed
2039
            pmeUpdateBsplinesKernel.setArg<cl::Buffer>(3, pmeAtomGridIndex.getDeviceBuffer());
2040
            pmeUpdateBsplinesKernel.setArg<cl::Buffer>(12, charges.getDeviceBuffer());
peastman's avatar
peastman committed
2041
2042
            pmeAtomRangeKernel.setArg<cl::Buffer>(0, pmeAtomGridIndex.getDeviceBuffer());
            pmeAtomRangeKernel.setArg<cl::Buffer>(1, pmeAtomRange.getDeviceBuffer());
2043
            pmeAtomRangeKernel.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2044
            pmeZIndexKernel.setArg<cl::Buffer>(0, pmeAtomGridIndex.getDeviceBuffer());
2045
            pmeZIndexKernel.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
2046
            pmeSpreadChargeKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2047
2048
            pmeSpreadChargeKernel.setArg<cl::Buffer>(1, pmeAtomGridIndex.getDeviceBuffer());
            pmeSpreadChargeKernel.setArg<cl::Buffer>(2, pmeAtomRange.getDeviceBuffer());
peastman's avatar
peastman committed
2049
            if (cl.getSupports64BitGlobalAtomics())
peastman's avatar
peastman committed
2050
                pmeSpreadChargeKernel.setArg<cl::Buffer>(3, pmeGrid2.getDeviceBuffer());
peastman's avatar
peastman committed
2051
            else
peastman's avatar
peastman committed
2052
2053
                pmeSpreadChargeKernel.setArg<cl::Buffer>(3, pmeGrid.getDeviceBuffer());
            pmeSpreadChargeKernel.setArg<cl::Buffer>(4, pmeBsplineTheta.getDeviceBuffer());
2054
2055
2056
2057
            if (deviceIsCpu || cl.getSupports64BitGlobalAtomics())
                pmeSpreadChargeKernel.setArg<cl::Buffer>(13, charges.getDeviceBuffer());
            else
                pmeSpreadChargeKernel.setArg<cl::Buffer>(5, charges.getDeviceBuffer());
peastman's avatar
peastman committed
2058
2059
2060
2061
2062
2063
2064
2065
2066
            pmeConvolutionKernel.setArg<cl::Buffer>(0, pmeGrid2.getDeviceBuffer());
            pmeConvolutionKernel.setArg<cl::Buffer>(1, pmeBsplineModuliX.getDeviceBuffer());
            pmeConvolutionKernel.setArg<cl::Buffer>(2, pmeBsplineModuliY.getDeviceBuffer());
            pmeConvolutionKernel.setArg<cl::Buffer>(3, pmeBsplineModuliZ.getDeviceBuffer());
            pmeEvalEnergyKernel.setArg<cl::Buffer>(0, pmeGrid2.getDeviceBuffer());
            pmeEvalEnergyKernel.setArg<cl::Buffer>(1, usePmeQueue ? pmeEnergyBuffer.getDeviceBuffer() : cl.getEnergyBuffer().getDeviceBuffer());
            pmeEvalEnergyKernel.setArg<cl::Buffer>(2, pmeBsplineModuliX.getDeviceBuffer());
            pmeEvalEnergyKernel.setArg<cl::Buffer>(3, pmeBsplineModuliY.getDeviceBuffer());
            pmeEvalEnergyKernel.setArg<cl::Buffer>(4, pmeBsplineModuliZ.getDeviceBuffer());
2067
2068
            pmeInterpolateForceKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
            pmeInterpolateForceKernel.setArg<cl::Buffer>(1, cl.getForceBuffers().getDeviceBuffer());
peastman's avatar
peastman committed
2069
2070
            pmeInterpolateForceKernel.setArg<cl::Buffer>(2, pmeGrid.getDeviceBuffer());
            pmeInterpolateForceKernel.setArg<cl::Buffer>(11, pmeAtomGridIndex.getDeviceBuffer());
2071
            pmeInterpolateForceKernel.setArg<cl::Buffer>(12, charges.getDeviceBuffer());
2072
2073
            if (cl.getSupports64BitGlobalAtomics()) {
                pmeFinishSpreadChargeKernel = cl::Kernel(program, "finishSpreadCharge");
peastman's avatar
peastman committed
2074
2075
                pmeFinishSpreadChargeKernel.setArg<cl::Buffer>(0, pmeGrid2.getDeviceBuffer());
                pmeFinishSpreadChargeKernel.setArg<cl::Buffer>(1, pmeGrid.getDeviceBuffer());
2076
            }
2077
2078
            if (usePmeQueue)
                syncQueue->setKernel(cl::Kernel(program, "addEnergy"));
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099

            if (doLJPME) {
                // Create kernels for LJ PME.

                pmeDefines["EWALD_ALPHA"] = cl.doubleToString(dispersionAlpha);
                pmeDefines["GRID_SIZE_X"] = cl.intToString(dispersionGridSizeX);
                pmeDefines["GRID_SIZE_Y"] = cl.intToString(dispersionGridSizeY);
                pmeDefines["GRID_SIZE_Z"] = cl.intToString(dispersionGridSizeZ);
                pmeDefines["EPSILON_FACTOR"] = "1";
                pmeDefines["RECIP_EXP_FACTOR"] = cl.doubleToString(M_PI*M_PI/(dispersionAlpha*dispersionAlpha));
                pmeDefines["USE_LJPME"] = "1";
                program = cl.createProgram(OpenCLKernelSources::pme, pmeDefines);
                pmeDispersionUpdateBsplinesKernel = cl::Kernel(program, "updateBsplines");
                pmeDispersionAtomRangeKernel = cl::Kernel(program, "findAtomRangeForGrid");
                pmeDispersionZIndexKernel = cl::Kernel(program, "recordZIndex");
                pmeDispersionSpreadChargeKernel = cl::Kernel(program, "gridSpreadCharge");
                pmeDispersionConvolutionKernel = cl::Kernel(program, "reciprocalConvolution");
                pmeDispersionEvalEnergyKernel = cl::Kernel(program, "gridEvaluateEnergy");
                pmeDispersionInterpolateForceKernel = cl::Kernel(program, "gridInterpolateForce");
                int elementSize = (cl.getUseDoublePrecision() ? sizeof(mm_double4) : sizeof(mm_float4));
                pmeDispersionUpdateBsplinesKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2100
                pmeDispersionUpdateBsplinesKernel.setArg<cl::Buffer>(1, pmeBsplineTheta.getDeviceBuffer());
2101
                pmeDispersionUpdateBsplinesKernel.setArg(2, OpenCLContext::ThreadBlockSize*PmeOrder*elementSize, NULL);
peastman's avatar
peastman committed
2102
2103
2104
2105
                pmeDispersionUpdateBsplinesKernel.setArg<cl::Buffer>(3, pmeAtomGridIndex.getDeviceBuffer());
                pmeDispersionUpdateBsplinesKernel.setArg<cl::Buffer>(12, sigmaEpsilon.getDeviceBuffer());
                pmeDispersionAtomRangeKernel.setArg<cl::Buffer>(0, pmeAtomGridIndex.getDeviceBuffer());
                pmeDispersionAtomRangeKernel.setArg<cl::Buffer>(1, pmeAtomRange.getDeviceBuffer());
2106
                pmeDispersionAtomRangeKernel.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2107
                pmeDispersionZIndexKernel.setArg<cl::Buffer>(0, pmeAtomGridIndex.getDeviceBuffer());
2108
2109
                pmeDispersionZIndexKernel.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
                pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
2110
2111
                pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(1, pmeAtomGridIndex.getDeviceBuffer());
                pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(2, pmeAtomRange.getDeviceBuffer());
2112
                if (cl.getSupports64BitGlobalAtomics())
peastman's avatar
peastman committed
2113
                    pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(3, pmeGrid2.getDeviceBuffer());
2114
                else
peastman's avatar
peastman committed
2115
2116
                    pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(3, pmeGrid.getDeviceBuffer());
                pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(4, pmeBsplineTheta.getDeviceBuffer());
peastman's avatar
peastman committed
2117
                if (deviceIsCpu || cl.getSupports64BitGlobalAtomics())
peastman's avatar
peastman committed
2118
                    pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(13, sigmaEpsilon.getDeviceBuffer());
peastman's avatar
peastman committed
2119
                else
peastman's avatar
peastman committed
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
                    pmeDispersionSpreadChargeKernel.setArg<cl::Buffer>(5, sigmaEpsilon.getDeviceBuffer());
                pmeDispersionConvolutionKernel.setArg<cl::Buffer>(0, pmeGrid2.getDeviceBuffer());
                pmeDispersionConvolutionKernel.setArg<cl::Buffer>(1, pmeDispersionBsplineModuliX.getDeviceBuffer());
                pmeDispersionConvolutionKernel.setArg<cl::Buffer>(2, pmeDispersionBsplineModuliY.getDeviceBuffer());
                pmeDispersionConvolutionKernel.setArg<cl::Buffer>(3, pmeDispersionBsplineModuliZ.getDeviceBuffer());
                pmeDispersionEvalEnergyKernel.setArg<cl::Buffer>(0, pmeGrid2.getDeviceBuffer());
                pmeDispersionEvalEnergyKernel.setArg<cl::Buffer>(1, usePmeQueue ? pmeEnergyBuffer.getDeviceBuffer() : cl.getEnergyBuffer().getDeviceBuffer());
                pmeDispersionEvalEnergyKernel.setArg<cl::Buffer>(2, pmeDispersionBsplineModuliX.getDeviceBuffer());
                pmeDispersionEvalEnergyKernel.setArg<cl::Buffer>(3, pmeDispersionBsplineModuliY.getDeviceBuffer());
                pmeDispersionEvalEnergyKernel.setArg<cl::Buffer>(4, pmeDispersionBsplineModuliZ.getDeviceBuffer());
2130
2131
                pmeDispersionInterpolateForceKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
                pmeDispersionInterpolateForceKernel.setArg<cl::Buffer>(1, cl.getForceBuffers().getDeviceBuffer());
peastman's avatar
peastman committed
2132
2133
2134
                pmeDispersionInterpolateForceKernel.setArg<cl::Buffer>(2, pmeGrid.getDeviceBuffer());
                pmeDispersionInterpolateForceKernel.setArg<cl::Buffer>(11, pmeAtomGridIndex.getDeviceBuffer());
                pmeDispersionInterpolateForceKernel.setArg<cl::Buffer>(12, sigmaEpsilon.getDeviceBuffer());
2135
2136
                if (cl.getSupports64BitGlobalAtomics()) {
                    pmeDispersionFinishSpreadChargeKernel = cl::Kernel(program, "finishSpreadCharge");
peastman's avatar
peastman committed
2137
2138
                    pmeDispersionFinishSpreadChargeKernel.setArg<cl::Buffer>(0, pmeGrid2.getDeviceBuffer());
                    pmeDispersionFinishSpreadChargeKernel.setArg<cl::Buffer>(1, pmeGrid.getDeviceBuffer());
2139
2140
                }
            }
2141
       }
2142
    }
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
    
    // Update particle and exception parameters.

    bool paramChanged = false;
    for (int i = 0; i < paramNames.size(); i++) {
        double value = context.getParameter(paramNames[i]);
        if (value != paramValues[i]) {
            paramValues[i] = value;;
            paramChanged = true;
        }
    }
    if (paramChanged) {
Peter Eastman's avatar
Peter Eastman committed
2155
        recomputeParams = true;
2156
2157
2158
2159
2160
2161
2162
2163
2164
        if (cl.getUseDoublePrecision())
            globalParams.upload(paramValues);
        else {
            vector<float> v(paramValues.size());
            for (int i = 0; i < v.size(); i++)
                v[i] = paramValues[i];
            globalParams.upload(v);
        }
    }
Peter Eastman's avatar
Peter Eastman committed
2165
2166
2167
2168
    double energy = (includeReciprocal ? ewaldSelfEnergy : 0.0);
    if (recomputeParams || hasOffsets) {
        computeParamsKernel.setArg<cl_int>(1, includeEnergy && includeReciprocal);
        cl.executeKernel(computeParamsKernel, cl.getPaddedNumAtoms());
2169
2170
2171
2172
2173
        if (usePmeQueue) {
            vector<cl::Event> events(1);
            cl.getQueue().enqueueMarker(&events[0]);
            pmeQueue.enqueueWaitForEvents(events);
        }
Peter Eastman's avatar
Peter Eastman committed
2174
        energy = 0.0; // The Ewald self energy was computed in the kernel.
2175
        recomputeParams = false;
Peter Eastman's avatar
Peter Eastman committed
2176
    }
2177
2178
2179
    
    // Do reciprocal space calculations.
    
peastman's avatar
peastman committed
2180
    if (cosSinSums.isInitialized() && includeReciprocal) {
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
        mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
        mm_double4 recipBoxSize = mm_double4(2*M_PI/boxSize.x, 2*M_PI/boxSize.y, 2*M_PI/boxSize.z, 0.0);
        double recipCoefficient = ONE_4PI_EPS0*4*M_PI/(boxSize.x*boxSize.y*boxSize.z);
        if (cl.getUseDoublePrecision()) {
            ewaldSumsKernel.setArg<mm_double4>(3, recipBoxSize);
            ewaldSumsKernel.setArg<cl_double>(4, recipCoefficient);
            ewaldForcesKernel.setArg<mm_double4>(3, recipBoxSize);
            ewaldForcesKernel.setArg<cl_double>(4, recipCoefficient);
        }
        else {
            ewaldSumsKernel.setArg<mm_float4>(3, mm_float4((float) recipBoxSize.x, (float) recipBoxSize.y, (float) recipBoxSize.z, 0));
            ewaldSumsKernel.setArg<cl_float>(4, (cl_float) recipCoefficient);
            ewaldForcesKernel.setArg<mm_float4>(3, mm_float4((float) recipBoxSize.x, (float) recipBoxSize.y, (float) recipBoxSize.z, 0));
            ewaldForcesKernel.setArg<cl_float>(4, (cl_float) recipCoefficient);
        }
peastman's avatar
peastman committed
2196
        cl.executeKernel(ewaldSumsKernel, cosSinSums.getSize());
2197
2198
        cl.executeKernel(ewaldForcesKernel, cl.getNumAtoms());
    }
peastman's avatar
peastman committed
2199
    if (pmeGrid.isInitialized() && includeReciprocal) {
2200
        if (usePmeQueue && !includeEnergy)
2201
            cl.setQueue(pmeQueue);
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
        
        // Invert the periodic box vectors.
        
        Vec3 boxVectors[3];
        cl.getPeriodicBoxVectors(boxVectors[0], boxVectors[1], boxVectors[2]);
        double determinant = boxVectors[0][0]*boxVectors[1][1]*boxVectors[2][2];
        double scale = 1.0/determinant;
        mm_double4 recipBoxVectors[3];
        recipBoxVectors[0] = mm_double4(boxVectors[1][1]*boxVectors[2][2]*scale, 0, 0, 0);
        recipBoxVectors[1] = mm_double4(-boxVectors[1][0]*boxVectors[2][2]*scale, boxVectors[0][0]*boxVectors[2][2]*scale, 0, 0);
        recipBoxVectors[2] = mm_double4((boxVectors[1][0]*boxVectors[2][1]-boxVectors[1][1]*boxVectors[2][0])*scale, -boxVectors[0][0]*boxVectors[2][1]*scale, boxVectors[0][0]*boxVectors[1][1]*scale, 0);
        mm_float4 recipBoxVectorsFloat[3];
        for (int i = 0; i < 3; i++)
            recipBoxVectorsFloat[i] = mm_float4((float) recipBoxVectors[i].x, (float) recipBoxVectors[i].y, (float) recipBoxVectors[i].z, 0);
        
        // Execute the reciprocal space kernels.

peastman's avatar
peastman committed
2219
2220
        if (hasCoulomb) {
            setPeriodicBoxArgs(cl, pmeUpdateBsplinesKernel, 4);
2221
            if (cl.getUseDoublePrecision()) {
peastman's avatar
peastman committed
2222
2223
2224
                pmeUpdateBsplinesKernel.setArg<mm_double4>(9, recipBoxVectors[0]);
                pmeUpdateBsplinesKernel.setArg<mm_double4>(10, recipBoxVectors[1]);
                pmeUpdateBsplinesKernel.setArg<mm_double4>(11, recipBoxVectors[2]);
2225
2226
            }
            else {
peastman's avatar
peastman committed
2227
2228
2229
                pmeUpdateBsplinesKernel.setArg<mm_float4>(9, recipBoxVectorsFloat[0]);
                pmeUpdateBsplinesKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[1]);
                pmeUpdateBsplinesKernel.setArg<mm_float4>(11, recipBoxVectorsFloat[2]);
2230
            }
peastman's avatar
peastman committed
2231
2232
            cl.executeKernel(pmeUpdateBsplinesKernel, cl.getNumAtoms());
            if (deviceIsCpu && !cl.getSupports64BitGlobalAtomics()) {
2233
                setPeriodicBoxArgs(cl, pmeSpreadChargeKernel, 5);
2234
                if (cl.getUseDoublePrecision()) {
2235
2236
2237
                    pmeSpreadChargeKernel.setArg<mm_double4>(10, recipBoxVectors[0]);
                    pmeSpreadChargeKernel.setArg<mm_double4>(11, recipBoxVectors[1]);
                    pmeSpreadChargeKernel.setArg<mm_double4>(12, recipBoxVectors[2]);
2238
2239
                }
                else {
2240
2241
2242
                    pmeSpreadChargeKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[0]);
                    pmeSpreadChargeKernel.setArg<mm_float4>(11, recipBoxVectorsFloat[1]);
                    pmeSpreadChargeKernel.setArg<mm_float4>(12, recipBoxVectorsFloat[2]);
2243
                }
peastman's avatar
peastman committed
2244
                cl.executeKernel(pmeSpreadChargeKernel, 2*cl.getDevice().getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>(), 1);
2245
            }
2246
            else {
peastman's avatar
peastman committed
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
                sort->sort(pmeAtomGridIndex);
                if (cl.getSupports64BitGlobalAtomics()) {
                    setPeriodicBoxArgs(cl, pmeSpreadChargeKernel, 5);
                    if (cl.getUseDoublePrecision()) {
                        pmeSpreadChargeKernel.setArg<mm_double4>(10, recipBoxVectors[0]);
                        pmeSpreadChargeKernel.setArg<mm_double4>(11, recipBoxVectors[1]);
                        pmeSpreadChargeKernel.setArg<mm_double4>(12, recipBoxVectors[2]);
                    }
                    else {
                        pmeSpreadChargeKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[0]);
                        pmeSpreadChargeKernel.setArg<mm_float4>(11, recipBoxVectorsFloat[1]);
                        pmeSpreadChargeKernel.setArg<mm_float4>(12, recipBoxVectorsFloat[2]);
                    }
                    cl.executeKernel(pmeSpreadChargeKernel, cl.getNumAtoms());
                    cl.executeKernel(pmeFinishSpreadChargeKernel, gridSizeX*gridSizeY*gridSizeZ);
                }
                else {
                    cl.executeKernel(pmeAtomRangeKernel, cl.getNumAtoms());
                    setPeriodicBoxSizeArg(cl, pmeZIndexKernel, 2);
                    if (cl.getUseDoublePrecision())
                        pmeZIndexKernel.setArg<mm_double4>(3, recipBoxVectors[2]);
                    else
                        pmeZIndexKernel.setArg<mm_float4>(3, recipBoxVectorsFloat[2]);
                    cl.executeKernel(pmeZIndexKernel, cl.getNumAtoms());
                    cl.executeKernel(pmeSpreadChargeKernel, cl.getNumAtoms());
                }
2273
            }
peastman's avatar
peastman committed
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
            fft->execFFT(pmeGrid, pmeGrid2, true);
            mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
            if (cl.getUseDoublePrecision()) {
                pmeConvolutionKernel.setArg<mm_double4>(4, recipBoxVectors[0]);
                pmeConvolutionKernel.setArg<mm_double4>(5, recipBoxVectors[1]);
                pmeConvolutionKernel.setArg<mm_double4>(6, recipBoxVectors[2]);
                pmeEvalEnergyKernel.setArg<mm_double4>(5, recipBoxVectors[0]);
                pmeEvalEnergyKernel.setArg<mm_double4>(6, recipBoxVectors[1]);
                pmeEvalEnergyKernel.setArg<mm_double4>(7, recipBoxVectors[2]);
            }
            else {
                pmeConvolutionKernel.setArg<mm_float4>(4, recipBoxVectorsFloat[0]);
                pmeConvolutionKernel.setArg<mm_float4>(5, recipBoxVectorsFloat[1]);
                pmeConvolutionKernel.setArg<mm_float4>(6, recipBoxVectorsFloat[2]);
                pmeEvalEnergyKernel.setArg<mm_float4>(5, recipBoxVectorsFloat[0]);
                pmeEvalEnergyKernel.setArg<mm_float4>(6, recipBoxVectorsFloat[1]);
                pmeEvalEnergyKernel.setArg<mm_float4>(7, recipBoxVectorsFloat[2]);
            }
            if (includeEnergy)
                cl.executeKernel(pmeEvalEnergyKernel, gridSizeX*gridSizeY*gridSizeZ);
            cl.executeKernel(pmeConvolutionKernel, gridSizeX*gridSizeY*gridSizeZ);
            fft->execFFT(pmeGrid2, pmeGrid, false);
            setPeriodicBoxArgs(cl, pmeInterpolateForceKernel, 3);
            if (cl.getUseDoublePrecision()) {
                pmeInterpolateForceKernel.setArg<mm_double4>(8, recipBoxVectors[0]);
                pmeInterpolateForceKernel.setArg<mm_double4>(9, recipBoxVectors[1]);
                pmeInterpolateForceKernel.setArg<mm_double4>(10, recipBoxVectors[2]);
            }
            else {
                pmeInterpolateForceKernel.setArg<mm_float4>(8, recipBoxVectorsFloat[0]);
                pmeInterpolateForceKernel.setArg<mm_float4>(9, recipBoxVectorsFloat[1]);
                pmeInterpolateForceKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[2]);
            }
            if (deviceIsCpu)
                cl.executeKernel(pmeInterpolateForceKernel, 2*cl.getDevice().getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>(), 1);
            else
                cl.executeKernel(pmeInterpolateForceKernel, cl.getNumAtoms());
2311
        }
2312
        
peastman's avatar
peastman committed
2313
        if (doLJPME && hasLJ) {
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
            setPeriodicBoxArgs(cl, pmeDispersionUpdateBsplinesKernel, 4);
            if (cl.getUseDoublePrecision()) {
                pmeDispersionUpdateBsplinesKernel.setArg<mm_double4>(9, recipBoxVectors[0]);
                pmeDispersionUpdateBsplinesKernel.setArg<mm_double4>(10, recipBoxVectors[1]);
                pmeDispersionUpdateBsplinesKernel.setArg<mm_double4>(11, recipBoxVectors[2]);
            }
            else {
                pmeDispersionUpdateBsplinesKernel.setArg<mm_float4>(9, recipBoxVectorsFloat[0]);
                pmeDispersionUpdateBsplinesKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[1]);
                pmeDispersionUpdateBsplinesKernel.setArg<mm_float4>(11, recipBoxVectorsFloat[2]);
            }
            cl.executeKernel(pmeDispersionUpdateBsplinesKernel, cl.getNumAtoms());
            if (deviceIsCpu && !cl.getSupports64BitGlobalAtomics()) {
peastman's avatar
peastman committed
2327
                cl.clearBuffer(pmeGrid);
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
                setPeriodicBoxArgs(cl, pmeDispersionSpreadChargeKernel, 5);
                if (cl.getUseDoublePrecision()) {
                    pmeDispersionSpreadChargeKernel.setArg<mm_double4>(10, recipBoxVectors[0]);
                    pmeDispersionSpreadChargeKernel.setArg<mm_double4>(11, recipBoxVectors[1]);
                    pmeDispersionSpreadChargeKernel.setArg<mm_double4>(12, recipBoxVectors[2]);
                }
                else {
                    pmeDispersionSpreadChargeKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[0]);
                    pmeDispersionSpreadChargeKernel.setArg<mm_float4>(11, recipBoxVectorsFloat[1]);
                    pmeDispersionSpreadChargeKernel.setArg<mm_float4>(12, recipBoxVectorsFloat[2]);
                }
                cl.executeKernel(pmeDispersionSpreadChargeKernel, 2*cl.getDevice().getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>(), 1);
            }
            else {
peastman's avatar
peastman committed
2342
                sort->sort(pmeAtomGridIndex);
2343
                if (cl.getSupports64BitGlobalAtomics()) {
peastman's avatar
peastman committed
2344
                    cl.clearBuffer(pmeGrid2);
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
                    setPeriodicBoxArgs(cl, pmeDispersionSpreadChargeKernel, 5);
                    if (cl.getUseDoublePrecision()) {
                        pmeDispersionSpreadChargeKernel.setArg<mm_double4>(10, recipBoxVectors[0]);
                        pmeDispersionSpreadChargeKernel.setArg<mm_double4>(11, recipBoxVectors[1]);
                        pmeDispersionSpreadChargeKernel.setArg<mm_double4>(12, recipBoxVectors[2]);
                    }
                    else {
                        pmeDispersionSpreadChargeKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[0]);
                        pmeDispersionSpreadChargeKernel.setArg<mm_float4>(11, recipBoxVectorsFloat[1]);
                        pmeDispersionSpreadChargeKernel.setArg<mm_float4>(12, recipBoxVectorsFloat[2]);
                    }
                    cl.executeKernel(pmeDispersionSpreadChargeKernel, cl.getNumAtoms());
                    cl.executeKernel(pmeDispersionFinishSpreadChargeKernel, gridSizeX*gridSizeY*gridSizeZ);
                }
                else {
peastman's avatar
peastman committed
2360
                    cl.clearBuffer(pmeGrid);
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
                    cl.executeKernel(pmeDispersionAtomRangeKernel, cl.getNumAtoms());
                    setPeriodicBoxSizeArg(cl, pmeDispersionZIndexKernel, 2);
                    if (cl.getUseDoublePrecision())
                        pmeDispersionZIndexKernel.setArg<mm_double4>(3, recipBoxVectors[2]);
                    else
                        pmeDispersionZIndexKernel.setArg<mm_float4>(3, recipBoxVectorsFloat[2]);
                    cl.executeKernel(pmeDispersionZIndexKernel, cl.getNumAtoms());
                    cl.executeKernel(pmeDispersionSpreadChargeKernel, cl.getNumAtoms());
                }
            }
peastman's avatar
peastman committed
2371
            dispersionFft->execFFT(pmeGrid, pmeGrid2, true);
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
            mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
            if (cl.getUseDoublePrecision()) {
                pmeDispersionConvolutionKernel.setArg<mm_double4>(4, recipBoxVectors[0]);
                pmeDispersionConvolutionKernel.setArg<mm_double4>(5, recipBoxVectors[1]);
                pmeDispersionConvolutionKernel.setArg<mm_double4>(6, recipBoxVectors[2]);
                pmeDispersionEvalEnergyKernel.setArg<mm_double4>(5, recipBoxVectors[0]);
                pmeDispersionEvalEnergyKernel.setArg<mm_double4>(6, recipBoxVectors[1]);
                pmeDispersionEvalEnergyKernel.setArg<mm_double4>(7, recipBoxVectors[2]);
            }
            else {
                pmeDispersionConvolutionKernel.setArg<mm_float4>(4, recipBoxVectorsFloat[0]);
                pmeDispersionConvolutionKernel.setArg<mm_float4>(5, recipBoxVectorsFloat[1]);
                pmeDispersionConvolutionKernel.setArg<mm_float4>(6, recipBoxVectorsFloat[2]);
                pmeDispersionEvalEnergyKernel.setArg<mm_float4>(5, recipBoxVectorsFloat[0]);
                pmeDispersionEvalEnergyKernel.setArg<mm_float4>(6, recipBoxVectorsFloat[1]);
                pmeDispersionEvalEnergyKernel.setArg<mm_float4>(7, recipBoxVectorsFloat[2]);
            }
            if (includeEnergy)
                cl.executeKernel(pmeDispersionEvalEnergyKernel, gridSizeX*gridSizeY*gridSizeZ);
            cl.executeKernel(pmeDispersionConvolutionKernel, gridSizeX*gridSizeY*gridSizeZ);
peastman's avatar
peastman committed
2392
            dispersionFft->execFFT(pmeGrid2, pmeGrid, false);
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
            setPeriodicBoxArgs(cl, pmeDispersionInterpolateForceKernel, 3);
            if (cl.getUseDoublePrecision()) {
                pmeDispersionInterpolateForceKernel.setArg<mm_double4>(8, recipBoxVectors[0]);
                pmeDispersionInterpolateForceKernel.setArg<mm_double4>(9, recipBoxVectors[1]);
                pmeDispersionInterpolateForceKernel.setArg<mm_double4>(10, recipBoxVectors[2]);
            }
            else {
                pmeDispersionInterpolateForceKernel.setArg<mm_float4>(8, recipBoxVectorsFloat[0]);
                pmeDispersionInterpolateForceKernel.setArg<mm_float4>(9, recipBoxVectorsFloat[1]);
                pmeDispersionInterpolateForceKernel.setArg<mm_float4>(10, recipBoxVectorsFloat[2]);
            }
            if (deviceIsCpu)
                cl.executeKernel(pmeDispersionInterpolateForceKernel, 2*cl.getDevice().getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>(), 1);
            else
                cl.executeKernel(pmeDispersionInterpolateForceKernel, cl.getNumAtoms());
        }
2409
2410
2411
2412
        if (usePmeQueue) {
            pmeQueue.enqueueMarker(&pmeSyncEvent);
            cl.restoreDefaultQueue();
        }
2413
    }
2414
    if (dispersionCoefficient != 0.0 && includeDirect) {
2415
        mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
2416
2417
2418
        energy += dispersionCoefficient/(boxSize.x*boxSize.y*boxSize.z);
    }
    return energy;
2419
2420
}

2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
void OpenCLCalcNonbondedForceKernel::copyParametersToContext(ContextImpl& context, const NonbondedForce& force) {
    // Make sure the new parameters are acceptable.
    
    if (force.getNumParticles() != cl.getNumAtoms())
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
    if (!hasCoulomb || !hasLJ) {
        for (int i = 0; i < force.getNumParticles(); i++) {
            double charge, sigma, epsilon;
            force.getParticleParameters(i, charge, sigma, epsilon);
            if (!hasCoulomb && charge != 0.0)
                throw OpenMMException("updateParametersInContext: The nonbonded force kernel does not include Coulomb interactions, because all charges were originally 0");
            if (!hasLJ && epsilon != 0.0)
                throw OpenMMException("updateParametersInContext: The nonbonded force kernel does not include Lennard-Jones interactions, because all epsilons were originally 0");
        }
    }
    vector<int> exceptions;
    for (int i = 0; i < force.getNumExceptions(); i++) {
        int particle1, particle2;
        double chargeProd, sigma, epsilon;
        force.getExceptionParameters(i, particle1, particle2, chargeProd, sigma, epsilon);
2441
        if (exceptionAtoms.size() > exceptions.size() && make_pair(particle1, particle2) == exceptionAtoms[exceptions.size()])
2442
            exceptions.push_back(i);
2443
2444
        else if (chargeProd != 0.0 || epsilon != 0.0)
            throw OpenMMException("updateParametersInContext: The set of non-excluded exceptions has changed");
2445
2446
2447
2448
2449
2450
2451
2452
    }
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*exceptions.size()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*exceptions.size()/numContexts;
    int numExceptions = endIndex-startIndex;
    
    // Record the per-particle parameters.
    
2453
    vector<mm_float4> baseParticleParamVec(cl.getPaddedNumAtoms(), mm_float4(0, 0, 0, 0));
2454
2455
    for (int i = 0; i < force.getNumParticles(); i++) {
        double charge, sigma, epsilon;
2456
        force.getParticleParameters(i, charge, sigma, epsilon);
2457
        baseParticleParamVec[i] = mm_float4(charge, sigma, epsilon, 0);
2458
    }
2459
    baseParticleParams.upload(baseParticleParamVec);
2460
2461
2462
2463
2464
    
    // Record the exceptions.
    
    if (numExceptions > 0) {
        vector<vector<int> > atoms(numExceptions, vector<int>(2));
2465
        vector<mm_float4> baseExceptionParamsVec(numExceptions);
2466
2467
2468
        for (int i = 0; i < numExceptions; i++) {
            double chargeProd, sigma, epsilon;
            force.getExceptionParameters(exceptions[startIndex+i], atoms[i][0], atoms[i][1], chargeProd, sigma, epsilon);
2469
            baseExceptionParamsVec[i] = mm_float4(chargeProd, sigma, epsilon, 0);
2470
        }
2471
        baseExceptionParams.upload(baseExceptionParamsVec);
2472
2473
2474
2475
    }
    
    // Compute other values.
    
2476
2477
2478
2479
2480
2481
2482
2483
    ewaldSelfEnergy = 0.0;
    if (nonbondedMethod == Ewald || nonbondedMethod == PME || nonbondedMethod == LJPME) {
        for (int i = 0; i < force.getNumParticles(); i++) {
            ewaldSelfEnergy -= baseParticleParamVec[i].x*baseParticleParamVec[i].x*ONE_4PI_EPS0*alpha/sqrt(M_PI);
            if (doLJPME)
                ewaldSelfEnergy += baseParticleParamVec[i].z*pow(baseParticleParamVec[i].y*dispersionAlpha, 6)/3.0;
        }
    }
2484
    if (force.getUseDispersionCorrection() && cl.getContextIndex() == 0 && (nonbondedMethod == CutoffPeriodic || nonbondedMethod == Ewald || nonbondedMethod == PME))
2485
        dispersionCoefficient = NonbondedForceImpl::calcDispersionCorrection(context.getSystem(), force);
2486
    cl.invalidateMolecules(info);
Peter Eastman's avatar
Peter Eastman committed
2487
    recomputeParams = true;
2488
2489
}

2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
void OpenCLCalcNonbondedForceKernel::getPMEParameters(double& alpha, int& nx, int& ny, int& nz) const {
    if (nonbondedMethod != PME)
        throw OpenMMException("getPMEParametersInContext: This Context is not using PME");
    if (cl.getPlatformData().useCpuPme)
        cpuPme.getAs<CalcPmeReciprocalForceKernel>().getPMEParameters(alpha, nx, ny, nz);
    else {
        alpha = this->alpha;
        nx = gridSizeX;
        ny = gridSizeY;
        nz = gridSizeZ;
    }
}

2503
void OpenCLCalcNonbondedForceKernel::getLJPMEParameters(double& alpha, int& nx, int& ny, int& nz) const {
2504
2505
2506
    if (nonbondedMethod != LJPME)
        throw OpenMMException("getPMEParametersInContext: This Context is not using PME");
    if (cl.getPlatformData().useCpuPme)
2507
2508
        //cpuPme.getAs<CalcPmeReciprocalForceKernel>().getLJPMEParameters(alpha, nx, ny, nz);
        throw OpenMMException("getPMEParametersInContext: CPUPME has not been implemented for LJPME yet.");
2509
    else {
2510
2511
2512
2513
        alpha = this->dispersionAlpha;
        nx = dispersionGridSizeX;
        ny = dispersionGridSizeY;
        nz = dispersionGridSizeZ;
2514
2515
2516
    }
}

2517
class OpenCLCalcCustomNonbondedForceKernel::ForceInfo : public OpenCLForceInfo {
2518
public:
2519
    ForceInfo(int requiredBuffers, const CustomNonbondedForce& force) : OpenCLForceInfo(requiredBuffers), force(force) {
2520
2521
2522
2523
2524
        if (force.getNumInteractionGroups() > 0) {
            groupsForParticle.resize(force.getNumParticles());
            for (int i = 0; i < force.getNumInteractionGroups(); i++) {
                set<int> set1, set2;
                force.getInteractionGroupParameters(i, set1, set2);
peastman's avatar
peastman committed
2525
2526
2527
2528
                for (int p : set1)
                    groupsForParticle[p].insert(2*i);
                for (int p : set2)
                    groupsForParticle[p].insert(2*i+1);
2529
2530
            }
        }
2531
2532
2533
2534
2535
2536
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        vector<double> params1;
        vector<double> params2;
        force.getParticleParameters(particle1, params1);
        force.getParticleParameters(particle2, params2);
2537
        for (int i = 0; i < (int) params1.size(); i++)
2538
2539
            if (params1[i] != params2[i])
                return false;
2540
2541
        if (groupsForParticle.size() > 0 && groupsForParticle[particle1] != groupsForParticle[particle2])
            return false;
2542
2543
2544
        return true;
    }
    int getNumParticleGroups() {
2545
        return force.getNumExclusions();
2546
    }
Peter Eastman's avatar
Peter Eastman committed
2547
    void getParticlesInGroup(int index, vector<int>& particles) {
2548
        int particle1, particle2;
2549
        force.getExclusionParticles(index, particle1, particle2);
2550
2551
2552
2553
2554
2555
2556
2557
2558
        particles.resize(2);
        particles[0] = particle1;
        particles[1] = particle2;
    }
    bool areGroupsIdentical(int group1, int group2) {
        return true;
    }
private:
    const CustomNonbondedForce& force;
2559
    vector<set<int> > groupsForParticle;
2560
2561
2562
2563
2564
};

OpenCLCalcCustomNonbondedForceKernel::~OpenCLCalcCustomNonbondedForceKernel() {
    if (params != NULL)
        delete params;
2565
2566
    if (forceCopy != NULL)
        delete forceCopy;
2567
2568
2569
2570
2571
2572
}

void OpenCLCalcCustomNonbondedForceKernel::initialize(const System& system, const CustomNonbondedForce& force) {
    int forceIndex;
    for (forceIndex = 0; forceIndex < system.getNumForces() && &system.getForce(forceIndex) != &force; ++forceIndex)
        ;
2573
    string prefix = (force.getNumInteractionGroups() == 0 ? "custom"+cl.intToString(forceIndex)+"_" : "");
2574
2575
2576
2577

    // Record parameters and exclusions.

    int numParticles = force.getNumParticles();
2578
    params = new OpenCLParameterSet(cl, force.getNumPerParticleParameters(), numParticles, "customNonbondedParameters");
2579
    if (force.getNumGlobalParameters() > 0)
peastman's avatar
peastman committed
2580
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customNonbondedGlobals", CL_MEM_READ_ONLY);
2581
    vector<vector<cl_float> > paramVector(numParticles);
2582
2583
2584
2585
    vector<vector<int> > exclusionList(numParticles);
    for (int i = 0; i < numParticles; i++) {
        vector<double> parameters;
        force.getParticleParameters(i, parameters);
2586
        paramVector[i].resize(parameters.size());
2587
        for (int j = 0; j < (int) parameters.size(); j++)
2588
            paramVector[i][j] = (cl_float) parameters[j];
2589
2590
        exclusionList[i].push_back(i);
    }
2591
2592
2593
2594
2595
    for (int i = 0; i < force.getNumExclusions(); i++) {
        int particle1, particle2;
        force.getExclusionParticles(i, particle1, particle2);
        exclusionList[particle1].push_back(particle2);
        exclusionList[particle2].push_back(particle1);
2596
    }
2597
    params->setParameterValues(paramVector);
2598
2599
2600

    // Record the tabulated functions.

2601
2602
    map<string, Lepton::CustomFunction*> functions;
    vector<pair<string, string> > functionDefinitions;
2603
    vector<const TabulatedFunction*> functionList;
2604
    vector<string> tableTypes;
peastman's avatar
peastman committed
2605
2606
    tabulatedFunctions.resize(force.getNumTabulatedFunctions());
    for (int i = 0; i < force.getNumTabulatedFunctions(); i++) {
2607
2608
        functionList.push_back(&force.getTabulatedFunction(i));
        string name = force.getTabulatedFunctionName(i);
2609
        string arrayName = prefix+"table"+cl.intToString(i);
2610
        functionDefinitions.push_back(make_pair(name, arrayName));
2611
        functions[name] = cl.getExpressionUtilities().getFunctionPlaceholder(force.getTabulatedFunction(i));
peastman's avatar
peastman committed
2612
        int width;
2613
        vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(force.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
2614
2615
2616
        tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
        tabulatedFunctions[i].upload(f);
        cl.getNonbondedUtilities().addArgument(OpenCLNonbondedUtilities::ParameterInfo(arrayName, "float", width, width*sizeof(float), tabulatedFunctions[i].getDeviceBuffer()));
2617
2618
2619
2620
        if (width == 1)
            tableTypes.push_back("float");
        else
            tableTypes.push_back("float"+cl.intToString(width));
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
    }

    // Record information for the expressions.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
peastman's avatar
peastman committed
2631
2632
    if (globals.isInitialized())
        globals.upload(globalParamValues);
2633
2634
    bool useCutoff = (force.getNonbondedMethod() != CustomNonbondedForce::NoCutoff);
    bool usePeriodic = (force.getNonbondedMethod() != CustomNonbondedForce::NoCutoff && force.getNonbondedMethod() != CustomNonbondedForce::CutoffNonPeriodic);
2635
    Lepton::ParsedExpression energyExpression = Lepton::Parser::parse(force.getEnergyFunction(), functions).optimize();
2636
    Lepton::ParsedExpression forceExpression = energyExpression.differentiate("r").optimize();
2637
    map<string, Lepton::ParsedExpression> forceExpressions;
2638
    forceExpressions["real customEnergy = "] = energyExpression;
2639
    forceExpressions["tempForce -= "] = forceExpression;
2640
2641
2642

    // Create the kernels.

2643
2644
2645
2646
2647
    vector<pair<ExpressionTreeNode, string> > variables;
    ExpressionTreeNode rnode(new Operation::Variable("r"));
    variables.push_back(make_pair(rnode, "r"));
    variables.push_back(make_pair(ExpressionTreeNode(new Operation::Square(), rnode), "r2"));
    variables.push_back(make_pair(ExpressionTreeNode(new Operation::Reciprocal(), rnode), "invR"));
2648
2649
    for (int i = 0; i < force.getNumPerParticleParameters(); i++) {
        const string& name = force.getPerParticleParameterName(i);
2650
2651
        variables.push_back(makeVariable(name+"1", prefix+"params"+params->getParameterSuffix(i, "1")));
        variables.push_back(makeVariable(name+"2", prefix+"params"+params->getParameterSuffix(i, "2")));
2652
2653
2654
    }
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        const string& name = force.getGlobalParameterName(i);
2655
        string value = "globals["+cl.intToString(i)+"]";
2656
        variables.push_back(makeVariable(name, prefix+value));
2657
    }
2658
2659
2660
2661
2662
2663
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string paramName = force.getEnergyParameterDerivativeName(i);
        string derivVariable = cl.getNonbondedUtilities().addEnergyParameterDerivative(paramName);
        Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
        forceExpressions[derivVariable+" += interactionScale*switchValue*"] = derivExpression;
    }
2664
    stringstream compute;
2665
    compute << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, prefix+"temp");
2666
2667
    map<string, string> replacements;
    replacements["COMPUTE_FORCE"] = compute.str();
2668
2669
2670
2671
2672
2673
2674
2675
2676
    replacements["USE_SWITCH"] = (useCutoff && force.getUseSwitchingFunction() ? "1" : "0");
    if (force.getUseSwitchingFunction()) {
        // Compute the switching coefficients.
        
        replacements["SWITCH_CUTOFF"] = cl.doubleToString(force.getSwitchingDistance());
        replacements["SWITCH_C3"] = cl.doubleToString(10/pow(force.getSwitchingDistance()-force.getCutoffDistance(), 3.0));
        replacements["SWITCH_C4"] = cl.doubleToString(15/pow(force.getSwitchingDistance()-force.getCutoffDistance(), 4.0));
        replacements["SWITCH_C5"] = cl.doubleToString(6/pow(force.getSwitchingDistance()-force.getCutoffDistance(), 5.0));
    }
2677
    string source = cl.replaceStrings(OpenCLKernelSources::customNonbonded, replacements);
2678
    if (force.getNumInteractionGroups() > 0)
2679
        initInteractionGroups(force, source, tableTypes);
2680
2681
2682
2683
2684
2685
    else {
        cl.getNonbondedUtilities().addInteraction(useCutoff, usePeriodic, true, force.getCutoffDistance(), exclusionList, source, force.getForceGroup());
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
            cl.getNonbondedUtilities().addParameter(OpenCLNonbondedUtilities::ParameterInfo(prefix+"params"+cl.intToString(i+1), buffer.getComponentType(), buffer.getNumComponents(), buffer.getSize(), buffer.getMemory()));
        }
peastman's avatar
peastman committed
2686
2687
2688
        if (globals.isInitialized()) {
            globals.upload(globalParamValues);
            cl.getNonbondedUtilities().addArgument(OpenCLNonbondedUtilities::ParameterInfo(prefix+"globals", "float", 1, sizeof(cl_float), globals.getDeviceBuffer()));
2689
        }
2690
    }
2691
2692
    info = new ForceInfo(cl.getNonbondedUtilities().getNumForceBuffers(), force);
    cl.addForce(info);
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
    
    // Record information for the long range correction.
    
    if (force.getNonbondedMethod() == CustomNonbondedForce::CutoffPeriodic && force.getUseLongRangeCorrection() && cl.getContextIndex() == 0) {
        forceCopy = new CustomNonbondedForce(force);
        hasInitializedLongRangeCorrection = false;
    }
    else {
        longRangeCoefficient = 0.0;
        hasInitializedLongRangeCorrection = true;
    }
2704
2705
}

2706
void OpenCLCalcCustomNonbondedForceKernel::initInteractionGroups(const CustomNonbondedForce& force, const string& interactionSource, const vector<string>& tableTypes) {
2707
2708
2709
2710
    // Process groups to form tiles.
    
    vector<vector<int> > atomLists;
    vector<pair<int, int> > tiles;
2711
2712
    vector<int> tileGroup;
    vector<vector<int> > duplicateAtomsForGroup;
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
    for (int group = 0; group < force.getNumInteractionGroups(); group++) {
        // Get the list of atoms in this group and sort them.
        
        set<int> set1, set2;
        force.getInteractionGroupParameters(group, set1, set2);
        vector<int> atoms1, atoms2;
        atoms1.insert(atoms1.begin(), set1.begin(), set1.end());
        atoms2.insert(atoms2.begin(), set2.begin(), set2.end());
        sort(atoms1.begin(), atoms1.end());
        sort(atoms2.begin(), atoms2.end());
2723
2724
2725
2726
        duplicateAtomsForGroup.push_back(vector<int>());
        set_intersection(set1.begin(), set1.end(), set2.begin(), set2.end(),
                inserter(duplicateAtomsForGroup[group], duplicateAtomsForGroup[group].begin()));
        sort(duplicateAtomsForGroup[group].begin(), duplicateAtomsForGroup[group].end());
2727
2728
2729
2730
        
        // Find how many tiles we will create for this group.
        
        int tileWidth = min(min(32, (int) atoms1.size()), (int) atoms2.size());
2731
2732
        if (tileWidth == 0)
            continue;
2733
2734
2735
2736
2737
        int numBlocks1 = (atoms1.size()+tileWidth-1)/tileWidth;
        int numBlocks2 = (atoms2.size()+tileWidth-1)/tileWidth;
        
        // Add the tiles.
        
2738
        int firstTile = tiles.size();
2739
        for (int i = 0; i < numBlocks1; i++)
2740
            for (int j = 0; j < numBlocks2; j++) {
2741
                tiles.push_back(make_pair(atomLists.size()+i, atomLists.size()+numBlocks1+j));
2742
2743
                tileGroup.push_back(group);
            }
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
        
        // Add the atom lists.
        
        for (int i = 0; i < numBlocks1; i++) {
            vector<int> atoms;
            int first = i*tileWidth;
            int last = min((i+1)*tileWidth, (int) atoms1.size());
            for (int j = first; j < last; j++)
                atoms.push_back(atoms1[j]);
            atomLists.push_back(atoms);
        }
        for (int i = 0; i < numBlocks2; i++) {
            vector<int> atoms;
            int first = i*tileWidth;
            int last = min((i+1)*tileWidth, (int) atoms2.size());
            for (int j = first; j < last; j++)
                atoms.push_back(atoms2[j]);
            atomLists.push_back(atoms);
        }
    }
    
    // Build a lookup table for quickly identifying excluded interactions.
    
2767
    vector<set<int> > exclusions(force.getNumParticles());
2768
2769
2770
    for (int i = 0; i < force.getNumExclusions(); i++) {
        int p1, p2;
        force.getExclusionParticles(i, p1, p2);
2771
2772
        exclusions[p1].insert(p2);
        exclusions[p2].insert(p1);
2773
2774
2775
2776
2777
2778
2779
2780
    }
    
    // Build the exclusion flags for each tile.  While we're at it, filter out tiles
    // where all interactions are excluded, and sort the tiles by size.

    vector<vector<int> > exclusionFlags(tiles.size());
    vector<pair<int, int> > tileOrder;
    for (int tile = 0; tile < tiles.size(); tile++) {
2781
        bool swapped = false;
2782
2783
2784
2785
2786
2787
        if (atomLists[tiles[tile].first].size() < atomLists[tiles[tile].second].size()) {
            // For efficiency, we want the first axis to be the larger one.
            
            int swap = tiles[tile].first;
            tiles[tile].first = tiles[tile].second;
            tiles[tile].second = swap;
2788
            swapped = true;
2789
2790
2791
        }
        vector<int>& atoms1 = atomLists[tiles[tile].first];
        vector<int>& atoms2 = atomLists[tiles[tile].second];
2792
        vector<int>& duplicateAtoms = duplicateAtomsForGroup[tileGroup[tile]];
2793
2794
        vector<int>& flags = exclusionFlags[tile];
        flags.resize(atoms1.size(), (int) (1LL<<atoms2.size())-1);
2795
        int numExcluded = 0;
2796
2797
2798
        for (int i = 0; i < (int) atoms1.size(); i++) {
            int a1 = atoms1[i];
            bool a1IsDuplicate = binary_search(duplicateAtoms.begin(), duplicateAtoms.end(), a1);
2799
2800
            for (int j = 0; j < (int) atoms2.size(); j++) {
                int a2 = atoms2[j];
peastman's avatar
peastman committed
2801
                bool isExcluded = false;
2802
                if (a1 == a2 || exclusions[a1].find(a2) != exclusions[a1].end())
peastman's avatar
peastman committed
2803
                    isExcluded = true; // This is an excluded interaction.
2804
2805
                else if ((a1 > a2) == swapped && a1IsDuplicate && binary_search(duplicateAtoms.begin(), duplicateAtoms.end(), a2))
                    isExcluded = true; // Both atoms are in both sets, so skip duplicate interactions.
peastman's avatar
peastman committed
2806
                if (isExcluded) {
2807
2808
2809
2810
                    flags[i] &= -1-(1<<j);
                    numExcluded++;
                }
            }
2811
        }
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
        if (numExcluded == atoms1.size()*atoms2.size())
            continue; // All interactions are excluded.
        tileOrder.push_back(make_pair((int) -atoms2.size(), tile));
    }
    sort(tileOrder.begin(), tileOrder.end());
    
    // Merge tiles to get as close as possible to 32 along the first axis of each one.
    
    vector<int> tileSetStart;
    tileSetStart.push_back(0);
    int tileSetSize = 0;
    for (int i = 0; i < tileOrder.size(); i++) {
        int tile = tileOrder[i].second;
        int size = atomLists[tiles[tile].first].size();
        if (tileSetSize+size > 32) {
            tileSetStart.push_back(i);
            tileSetSize = 0;
        }
        tileSetSize += size;
    }
    tileSetStart.push_back(tileOrder.size());
    
    // Build the data structures.
    
    int numTileSets = tileSetStart.size()-1;
    vector<mm_int4> groupData;
    for (int tileSet = 0; tileSet < numTileSets; tileSet++) {
        int indexInTileSet = 0;
2840
2841
2842
2843
2844
2845
2846
2847
        int minSize = 0;
        if (cl.getSIMDWidth() < 32) {
            // We need to include a barrier inside the inner loop, so ensure that all
            // threads will loop the same number of times.
            
            for (int i = tileSetStart[tileSet]; i < tileSetStart[tileSet+1]; i++)
                minSize = max(minSize, (int) atomLists[tiles[tileOrder[i].second].first].size());
        }
2848
2849
2850
2851
        for (int i = tileSetStart[tileSet]; i < tileSetStart[tileSet+1]; i++) {
            int tile = tileOrder[i].second;
            vector<int>& atoms1 = atomLists[tiles[tile].first];
            vector<int>& atoms2 = atomLists[tiles[tile].second];
2852
            int range = indexInTileSet + ((indexInTileSet+max(minSize, (int) atoms1.size()))<<16);
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
            int allFlags = (1<<atoms2.size())-1;
            for (int j = 0; j < (int) atoms1.size(); j++) {
                int a1 = atoms1[j];
                int a2 = (j < atoms2.size() ? atoms2[j] : 0);
                int flags = (exclusionFlags[tile].size() > 0 ? exclusionFlags[tile][j] : allFlags);
                groupData.push_back(mm_int4(a1, a2, range, flags<<indexInTileSet));
            }
            indexInTileSet += atoms1.size();
        }
        for (; indexInTileSet < 32; indexInTileSet++)
2863
            groupData.push_back(mm_int4(0, 0, minSize<<16, 0));
2864
    }
peastman's avatar
peastman committed
2865
2866
    interactionGroupData.initialize<mm_int4>(cl, groupData.size(), "interactionGroupData");
    interactionGroupData.upload(groupData);
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
    numGroupTiles.initialize<cl_int>(cl, 1, "numGroupTiles");

    // Allocate space for a neighbor list, if necessary.

    if (force.getNonbondedMethod() != CustomNonbondedForce::NoCutoff && groupData.size() > cl.getNumThreadBlocks()) {
        filteredGroupData.initialize<mm_int4>(cl, groupData.size(), "filteredGroupData");
        interactionGroupData.copyTo(filteredGroupData);
        int numTiles = groupData.size()/32;
        numGroupTiles.upload(&numTiles);
    }
2877
2878
2879
    
    // Create the kernel.
    
2880
    hasParamDerivs = (force.getNumEnergyParameterDerivatives() > 0);
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
    map<string, string> replacements;
    replacements["COMPUTE_INTERACTION"] = interactionSource;
    const string suffixes[] = {"x", "y", "z", "w"};
    stringstream localData;
    int localDataSize = 0;
    vector<OpenCLNonbondedUtilities::ParameterInfo>& buffers = params->getBuffers(); 
    for (int i = 0; i < (int) buffers.size(); i++) {
        if (buffers[i].getNumComponents() == 1)
            localData<<buffers[i].getComponentType()<<" params"<<(i+1)<<";\n";
        else {
            for (int j = 0; j < buffers[i].getNumComponents(); ++j)
                localData<<buffers[i].getComponentType()<<" params"<<(i+1)<<"_"<<suffixes[j]<<";\n";
        }
        localDataSize += buffers[i].getSize();
    }
    replacements["ATOM_PARAMETER_DATA"] = localData.str();
    stringstream args;
    for (int i = 0; i < (int) buffers.size(); i++)
        args<<", __global const "<<buffers[i].getType()<<"* restrict global_params"<<(i+1);
2900
2901
    for (int i = 0; i < (int) tabulatedFunctions.size(); i++)
        args << ", __global const " << tableTypes[i]<< "* restrict table" << i;
peastman's avatar
peastman committed
2902
    if (globals.isInitialized())
2903
        args<<", __global const float* restrict globals";
2904
2905
    if (hasParamDerivs)
        args << ", __global mixed* restrict energyParamDerivs";
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
    replacements["PARAMETER_ARGUMENTS"] = args.str();
    stringstream load1;
    for (int i = 0; i < (int) buffers.size(); i++)
        load1<<buffers[i].getType()<<" params"<<(i+1)<<"1 = global_params"<<(i+1)<<"[atom1];\n";
    replacements["LOAD_ATOM1_PARAMETERS"] = load1.str();
    stringstream loadLocal2;
    for (int i = 0; i < (int) buffers.size(); i++) {
        if (buffers[i].getNumComponents() == 1)
            loadLocal2<<"localData[get_local_id(0)].params"<<(i+1)<<" = global_params"<<(i+1)<<"[atom2];\n";
        else {
            loadLocal2<<buffers[i].getType()<<" temp_params"<<(i+1)<<" = global_params"<<(i+1)<<"[atom2];\n";
            for (int j = 0; j < buffers[i].getNumComponents(); ++j)
                loadLocal2<<"localData[get_local_id(0)].params"<<(i+1)<<"_"<<suffixes[j]<<" = temp_params"<<(i+1)<<"."<<suffixes[j]<<";\n";
        }
    }
    replacements["LOAD_LOCAL_PARAMETERS"] = loadLocal2.str();
    stringstream load2;
    for (int i = 0; i < (int) buffers.size(); i++) {
        if (buffers[i].getNumComponents() == 1)
            load2<<buffers[i].getType()<<" params"<<(i+1)<<"2 = localData[localIndex].params"<<(i+1)<<";\n";
        else {
2927
            load2<<buffers[i].getType()<<" params"<<(i+1)<<"2 = ("<<buffers[i].getType()<<") (";
2928
2929
2930
2931
2932
2933
2934
2935
2936
            for (int j = 0; j < buffers[i].getNumComponents(); ++j) {
                if (j > 0)
                    load2<<", ";
                load2<<"localData[localIndex].params"<<(i+1)<<"_"<<suffixes[j];
            }
            load2<<");\n";
        }
    }
    replacements["LOAD_ATOM2_PARAMETERS"] = load2.str();
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
    stringstream initDerivs, saveDerivs;
    const vector<string>& allParamDerivNames = cl.getEnergyParamDerivNames();
    int numDerivs = allParamDerivNames.size();
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string paramName = force.getEnergyParameterDerivativeName(i);
        string derivVariable = cl.getNonbondedUtilities().addEnergyParameterDerivative(paramName);
        initDerivs<<"mixed "<<derivVariable<<" = 0;\n";
        for (int index = 0; index < numDerivs; index++)
            if (allParamDerivNames[index] == paramName)
                saveDerivs<<"energyParamDerivs[get_global_id(0)*"<<numDerivs<<"+"<<index<<"] += "<<derivVariable<<";\n";
    }
    replacements["INIT_DERIVATIVES"] = initDerivs.str();
    replacements["SAVE_DERIVATIVES"] = saveDerivs.str();
2950
2951
2952
2953
2954
    map<string, string> defines;
    if (force.getNonbondedMethod() != CustomNonbondedForce::NoCutoff)
        defines["USE_CUTOFF"] = "1";
    if (force.getNonbondedMethod() == CustomNonbondedForce::CutoffPeriodic)
        defines["USE_PERIODIC"] = "1";
2955
2956
2957
    int localMemorySize = max(32, cl.getNonbondedUtilities().getForceThreadBlockSize());
    defines["LOCAL_MEMORY_SIZE"] = cl.intToString(localMemorySize);
    defines["WARPS_IN_BLOCK"] = cl.intToString(localMemorySize/32);
2958
2959
    double cutoff = force.getCutoffDistance();
    defines["CUTOFF_SQUARED"] = cl.doubleToString(cutoff*cutoff);
2960
2961
    double paddedCutoff = cl.getNonbondedUtilities().padCutoff(cutoff);
    defines["PADDED_CUTOFF_SQUARED"] = cl.doubleToString(paddedCutoff*paddedCutoff);
2962
2963
    defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
    defines["TILE_SIZE"] = "32";
2964
    defines["NUM_TILES"] = cl.intToString(numTileSets);
2965
2966
2967
2968
2969
2970
2971
2972
2973
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*numTileSets/numContexts;
    int endIndex = (cl.getContextIndex()+1)*numTileSets/numContexts;
    defines["FIRST_TILE"] = cl.intToString(startIndex);
    defines["LAST_TILE"] = cl.intToString(endIndex);
    if ((localDataSize/4)%2 == 0 && !cl.getUseDoublePrecision())
        defines["PARAMETER_SIZE_IS_EVEN"] = "1";
    cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customNonbondedGroups, replacements), defines);
    interactionGroupKernel = cl::Kernel(program, "computeInteractionGroups");
2974
2975
    prepareNeighborListKernel = cl::Kernel(program, "prepareToBuildNeighborList");
    buildNeighborListKernel = cl::Kernel(program, "buildNeighborList");
2976
2977
2978
    numGroupThreadBlocks = cl.getNonbondedUtilities().getNumForceThreadBlocks();
}

2979
double OpenCLCalcCustomNonbondedForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
2980
2981
2982
2983
2984
    useNeighborList = (filteredGroupData.isInitialized() && cl.getNonbondedUtilities().getUseCutoff());
    if (useNeighborList && cl.getContextIndex() > 0) {
        // When using a neighbor list, run the whole calculation on a single device.
        return 0.0;
    }
peastman's avatar
peastman committed
2985
    if (globals.isInitialized()) {
2986
        bool changed = false;
2987
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
2988
2989
2990
2991
2992
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
2993
        if (changed) {
peastman's avatar
peastman committed
2994
            globals.upload(globalParamValues);
2995
            if (forceCopy != NULL) {
2996
                CustomNonbondedForceImpl::calcLongRangeCorrection(*forceCopy, context.getOwner(), longRangeCoefficient, longRangeCoefficientDerivs);
2997
2998
2999
                hasInitializedLongRangeCorrection = true;
            }
        }
3000
    }
3001
    if (!hasInitializedLongRangeCorrection) {
3002
        CustomNonbondedForceImpl::calcLongRangeCorrection(*forceCopy, context.getOwner(), longRangeCoefficient, longRangeCoefficientDerivs);
3003
3004
        hasInitializedLongRangeCorrection = true;
    }
peastman's avatar
peastman committed
3005
    if (interactionGroupData.isInitialized()) {
3006
3007
3008
        if (!hasInitializedKernel) {
            hasInitializedKernel = true;
            int index = 0;
3009
3010
            bool useLong = cl.getSupports64BitGlobalAtomics();
            interactionGroupKernel.setArg<cl::Buffer>(index++, (useLong ? cl.getLongForceBuffer() : cl.getForceBuffers()).getDeviceBuffer());
3011
3012
            interactionGroupKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
            interactionGroupKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
3013
3014
            interactionGroupKernel.setArg<cl::Buffer>(index++, (useNeighborList ? filteredGroupData : interactionGroupData).getDeviceBuffer());
            interactionGroupKernel.setArg<cl::Buffer>(index++, numGroupTiles.getDeviceBuffer());
3015
            interactionGroupKernel.setArg<cl_int>(index++, useNeighborList);
3016
            index += 5;
peastman's avatar
peastman committed
3017
3018
            for (auto& buffer : params->getBuffers())
                interactionGroupKernel.setArg<cl::Memory>(index++, buffer.getMemory());
peastman's avatar
peastman committed
3019
3020
3021
3022
            for (auto& function : tabulatedFunctions)
                interactionGroupKernel.setArg<cl::Memory>(index++, function.getDeviceBuffer());
            if (globals.isInitialized())
                interactionGroupKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
3023
3024
            if (hasParamDerivs)
                interactionGroupKernel.setArg<cl::Memory>(index++, cl.getEnergyParamDerivBuffer().getDeviceBuffer());
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
            if (useNeighborList) {
                // Initialize kernels for building the interaction group neighbor list.
                
                prepareNeighborListKernel.setArg<cl::Buffer>(0, cl.getNonbondedUtilities().getRebuildNeighborList().getDeviceBuffer());
                prepareNeighborListKernel.setArg<cl::Buffer>(1, numGroupTiles.getDeviceBuffer());
                buildNeighborListKernel.setArg<cl::Buffer>(0, cl.getNonbondedUtilities().getRebuildNeighborList().getDeviceBuffer());
                buildNeighborListKernel.setArg<cl::Buffer>(1, numGroupTiles.getDeviceBuffer());
                buildNeighborListKernel.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
                buildNeighborListKernel.setArg<cl::Buffer>(3, interactionGroupData.getDeviceBuffer());
                buildNeighborListKernel.setArg<cl::Buffer>(4, filteredGroupData.getDeviceBuffer());
            }
3036
        }
3037
        int forceThreadBlockSize = max(32, cl.getNonbondedUtilities().getForceThreadBlockSize());
3038
3039
3040
3041
3042
3043
3044
3045
        if (useNeighborList) {
            // Rebuild the neighbor list, if necessary.

            setPeriodicBoxArgs(cl, buildNeighborListKernel, 5);
            cl.executeKernel(prepareNeighborListKernel, 1, 1);
            cl.executeKernel(buildNeighborListKernel, numGroupThreadBlocks*forceThreadBlockSize, forceThreadBlockSize);
        }
        setPeriodicBoxArgs(cl, interactionGroupKernel, 6);
3046
3047
        cl.executeKernel(interactionGroupKernel, numGroupThreadBlocks*forceThreadBlockSize, forceThreadBlockSize);
    }
3048
    mm_double4 boxSize = cl.getPeriodicBoxSizeDouble();
3049
3050
3051
3052
3053
    double volume = boxSize.x*boxSize.y*boxSize.z;
    map<string, double>& derivs = cl.getEnergyParamDerivWorkspace();
    for (int i = 0; i < longRangeCoefficientDerivs.size(); i++)
        derivs[forceCopy->getEnergyParameterDerivativeName(i)] += longRangeCoefficientDerivs[i]/volume;
    return longRangeCoefficient/volume;
3054
}
Peter Eastman's avatar
Peter Eastman committed
3055

3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
void OpenCLCalcCustomNonbondedForceKernel::copyParametersToContext(ContextImpl& context, const CustomNonbondedForce& force) {
    int numParticles = force.getNumParticles();
    if (numParticles != cl.getNumAtoms())
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
    
    // Record the per-particle parameters.
    
    vector<vector<cl_float> > paramVector(numParticles);
    vector<double> parameters;
    for (int i = 0; i < numParticles; i++) {
        force.getParticleParameters(i, parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
3073
3074
3075
    // If necessary, recompute the long range correction.
    
    if (forceCopy != NULL) {
3076
        CustomNonbondedForceImpl::calcLongRangeCorrection(force, context.getOwner(), longRangeCoefficient, longRangeCoefficientDerivs);
3077
3078
3079
3080
        hasInitializedLongRangeCorrection = true;
        *forceCopy = force;
    }
    
3081
3082
    // Mark that the current reordering may be invalid.
    
3083
    cl.invalidateMolecules(info);
3084
3085
}

3086
class OpenCLCalcGBSAOBCForceKernel::ForceInfo : public OpenCLForceInfo {
Peter Eastman's avatar
Peter Eastman committed
3087
public:
3088
    ForceInfo(int requiredBuffers, const GBSAOBCForce& force) : OpenCLForceInfo(requiredBuffers), force(force) {
Peter Eastman's avatar
Peter Eastman committed
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        double charge1, charge2, radius1, radius2, scale1, scale2;
        force.getParticleParameters(particle1, charge1, radius1, scale1);
        force.getParticleParameters(particle2, charge2, radius2, scale2);
        return (charge1 == charge2 && radius1 == radius2 && scale1 == scale2);
    }
private:
    const GBSAOBCForce& force;
};

3100
void OpenCLCalcGBSAOBCForceKernel::initialize(const System& system, const GBSAOBCForce& force) {
3101
3102
    if (cl.getPlatformData().contexts.size() > 1)
        throw OpenMMException("GBSAOBCForce does not support using multiple OpenCL devices");
3103
3104
3105
3106
    int forceIndex;
    for (forceIndex = 0; forceIndex < system.getNumForces() && &system.getForce(forceIndex) != &force; ++forceIndex)
        ;
    string prefix = "obc"+cl.intToString(forceIndex)+"_";
3107
    OpenCLNonbondedUtilities& nb = cl.getNonbondedUtilities();
peastman's avatar
peastman committed
3108
    params.initialize<mm_float2>(cl, cl.getPaddedNumAtoms(), "gbsaObcParams");
3109
    int elementSize = (cl.getUseDoublePrecision() ? sizeof(cl_double) : sizeof(cl_float));
3110
    charges.initialize(cl, cl.getPaddedNumAtoms(), elementSize, "gbsaObcCharges");
peastman's avatar
peastman committed
3111
3112
    bornRadii.initialize(cl, cl.getPaddedNumAtoms(), elementSize, "bornRadii");
    obcChain.initialize(cl, cl.getPaddedNumAtoms(), elementSize, "obcChain");
3113
    if (cl.getSupports64BitGlobalAtomics()) {
peastman's avatar
peastman committed
3114
3115
3116
3117
3118
        longBornSum.initialize<cl_long>(cl, cl.getPaddedNumAtoms(), "longBornSum");
        longBornForce.initialize<cl_long>(cl, cl.getPaddedNumAtoms(), "longBornForce");
        bornForce.initialize(cl, cl.getPaddedNumAtoms(), elementSize, "bornForce");
        cl.addAutoclearBuffer(longBornSum);
        cl.addAutoclearBuffer(longBornForce);
3119
3120
    }
    else {
peastman's avatar
peastman committed
3121
3122
3123
3124
        bornSum.initialize(cl, cl.getPaddedNumAtoms()*nb.getNumForceBuffers(), elementSize, "bornSum");
        bornForce.initialize(cl, cl.getPaddedNumAtoms()*nb.getNumForceBuffers(), elementSize, "bornForce");
        cl.addAutoclearBuffer(bornSum);
        cl.addAutoclearBuffer(bornForce);
3125
    }
3126
    vector<double> chargeVec(cl.getPaddedNumAtoms());
3127
    vector<mm_float2> paramsVector(cl.getPaddedNumAtoms(), mm_float2(1,1));
3128
    const double dielectricOffset = 0.009;
3129
    for (int i = 0; i < force.getNumParticles(); i++) {
3130
3131
3132
        double charge, radius, scalingFactor;
        force.getParticleParameters(i, charge, radius, scalingFactor);
        radius -= dielectricOffset;
3133
        chargeVec[i] = charge;
3134
        paramsVector[i] = mm_float2((float) radius, (float) (scalingFactor*radius));
3135
    }
3136
    if (cl.getUseDoublePrecision())
3137
3138
3139
3140
3141
3142
3143
        charges.upload(chargeVec);
    else {
        vector<float> c(charges.getSize());
        for (int i = 0; i < c.size(); i++)
            c[i] = (float) chargeVec[i];
        charges.upload(c);
    }
peastman's avatar
peastman committed
3144
    params.upload(paramsVector);
3145
    prefactor = -ONE_4PI_EPS0*((1.0/force.getSoluteDielectric())-(1.0/force.getSolventDielectric()));
3146
    surfaceAreaFactor = -6.0*4*M_PI*force.getSurfaceAreaEnergy();
3147
3148
    bool useCutoff = (force.getNonbondedMethod() != GBSAOBCForce::NoCutoff);
    bool usePeriodic = (force.getNonbondedMethod() != GBSAOBCForce::NoCutoff && force.getNonbondedMethod() != GBSAOBCForce::CutoffNonPeriodic);
3149
    cutoff = force.getCutoffDistance();
3150
    string source = OpenCLKernelSources::gbsaObc2;
3151
3152
3153
3154
3155
3156
3157
3158
    map<string, string> replacements;
    replacements["CHARGE1"] = prefix+"charge1";
    replacements["CHARGE2"] = prefix+"charge2";
    replacements["OBC_PARAMS1"] = prefix+"obcParams1";
    replacements["OBC_PARAMS2"] = prefix+"obcParams2";
    replacements["BORN_FORCE1"] = prefix+"bornForce1";
    replacements["BORN_FORCE2"] = prefix+"bornForce2";
    source = cl.replaceStrings(source, replacements);
3159
    nb.addInteraction(useCutoff, usePeriodic, false, cutoff, vector<vector<int> >(), source, force.getForceGroup());
3160
3161
3162
    nb.addParameter(OpenCLNonbondedUtilities::ParameterInfo(prefix+"charge", "float", 1, sizeof(cl_float), charges.getDeviceBuffer()));;
    nb.addParameter(OpenCLNonbondedUtilities::ParameterInfo(prefix+"obcParams", "float", 2, sizeof(cl_float2), params.getDeviceBuffer()));;
    nb.addParameter(OpenCLNonbondedUtilities::ParameterInfo(prefix+"bornForce", "real", 1, elementSize, bornForce.getDeviceBuffer()));;
3163
3164
    info = new ForceInfo(nb.getNumForceBuffers(), force);
    cl.addForce(info);
3165
3166
}

3167
double OpenCLCalcGBSAOBCForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
3168
    OpenCLNonbondedUtilities& nb = cl.getNonbondedUtilities();
3169
    bool deviceIsCpu = (cl.getDevice().getInfo<CL_DEVICE_TYPE>() == CL_DEVICE_TYPE_CPU);
3170
3171
3172
3173
    if (!hasCreatedKernels) {
        // These Kernels cannot be created in initialize(), because the OpenCLNonbondedUtilities has not been initialized yet then.

        hasCreatedKernels = true;
3174
        maxTiles = (nb.getUseCutoff() ? nb.getInteractingTiles().getSize() : 0);
3175
3176
3177
3178
3179
        map<string, string> defines;
        if (nb.getUseCutoff())
            defines["USE_CUTOFF"] = "1";
        if (nb.getUsePeriodic())
            defines["USE_PERIODIC"] = "1";
3180
3181
        defines["CUTOFF_SQUARED"] = cl.doubleToString(cutoff*cutoff);
        defines["CUTOFF"] = cl.doubleToString(cutoff);
3182
        defines["PREFACTOR"] = cl.doubleToString(prefactor);
3183
        defines["SURFACE_AREA_FACTOR"] = cl.doubleToString(surfaceAreaFactor);
3184
3185
3186
3187
        defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
        defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
        defines["NUM_BLOCKS"] = cl.intToString(cl.getNumAtomBlocks());
        defines["FORCE_WORK_GROUP_SIZE"] = cl.intToString(nb.getForceThreadBlockSize());
3188
3189
3190
3191
3192
3193
3194
3195
        defines["TILE_SIZE"] = cl.intToString(OpenCLContext::TileSize);
        int numExclusionTiles = nb.getExclusionTiles().getSize();
        defines["NUM_TILES_WITH_EXCLUSIONS"] = cl.intToString(numExclusionTiles);
        int numContexts = cl.getPlatformData().contexts.size();
        int startExclusionIndex = cl.getContextIndex()*numExclusionTiles/numContexts;
        int endExclusionIndex = (cl.getContextIndex()+1)*numExclusionTiles/numContexts;
        defines["FIRST_EXCLUSION_TILE"] = cl.intToString(startExclusionIndex);
        defines["LAST_EXCLUSION_TILE"] = cl.intToString(endExclusionIndex);
3196
3197
3198
        string platformVendor = cl::Platform(cl.getDevice().getInfo<CL_DEVICE_PLATFORM>()).getInfo<CL_PLATFORM_VENDOR>();
        if (platformVendor == "Apple")
            defines["USE_APPLE_WORKAROUND"] = "1";
3199
3200
3201
3202
        string file;
        if (deviceIsCpu)
            file = OpenCLKernelSources::gbsaObc_cpu;
        else
3203
            file = OpenCLKernelSources::gbsaObc;
3204
        cl::Program program = cl.createProgram(file, defines);
3205
        bool useLong = cl.getSupports64BitGlobalAtomics();
3206
        int index = 0;
3207
        computeBornSumKernel = cl::Kernel(program, "computeBornSum");
peastman's avatar
peastman committed
3208
        computeBornSumKernel.setArg<cl::Buffer>(index++, (useLong ? longBornSum.getDeviceBuffer() : bornSum.getDeviceBuffer()));
3209
        computeBornSumKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
3210
        computeBornSumKernel.setArg<cl::Buffer>(index++, charges.getDeviceBuffer());
peastman's avatar
peastman committed
3211
        computeBornSumKernel.setArg<cl::Buffer>(index++, params.getDeviceBuffer());
3212
        if (nb.getUseCutoff()) {
3213
3214
            computeBornSumKernel.setArg<cl::Buffer>(index++, nb.getInteractingTiles().getDeviceBuffer());
            computeBornSumKernel.setArg<cl::Buffer>(index++, nb.getInteractionCount().getDeviceBuffer());
3215
            index += 5; // The periodic box size arguments are set when the kernel is executed.
3216
            computeBornSumKernel.setArg<cl_uint>(index++, maxTiles);
3217
            computeBornSumKernel.setArg<cl::Buffer>(index++, nb.getBlockCenters().getDeviceBuffer());
3218
            computeBornSumKernel.setArg<cl::Buffer>(index++, nb.getBlockBoundingBoxes().getDeviceBuffer());
3219
            computeBornSumKernel.setArg<cl::Buffer>(index++, nb.getInteractingAtoms().getDeviceBuffer());
3220
        }
3221
3222
        else
            computeBornSumKernel.setArg<cl_uint>(index++, cl.getNumAtomBlocks()*(cl.getNumAtomBlocks()+1)/2);
3223
        computeBornSumKernel.setArg<cl::Buffer>(index++, nb.getExclusionTiles().getDeviceBuffer());
3224
        force1Kernel = cl::Kernel(program, "computeGBSAForce1");
3225
        index = 0;
3226
        force1Kernel.setArg<cl::Buffer>(index++, (useLong ? cl.getLongForceBuffer().getDeviceBuffer() : cl.getForceBuffers().getDeviceBuffer()));
peastman's avatar
peastman committed
3227
        force1Kernel.setArg<cl::Buffer>(index++, (useLong ? longBornForce.getDeviceBuffer() : bornForce.getDeviceBuffer()));
3228
3229
        force1Kernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
        force1Kernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
3230
        force1Kernel.setArg<cl::Buffer>(index++, charges.getDeviceBuffer());
peastman's avatar
peastman committed
3231
        force1Kernel.setArg<cl::Buffer>(index++, bornRadii.getDeviceBuffer());
3232
        index++; // Whether to include energy.
3233
        if (nb.getUseCutoff()) {
3234
3235
            force1Kernel.setArg<cl::Buffer>(index++, nb.getInteractingTiles().getDeviceBuffer());
            force1Kernel.setArg<cl::Buffer>(index++, nb.getInteractionCount().getDeviceBuffer());
3236
            index += 5; // The periodic box size arguments are set when the kernel is executed.
3237
            force1Kernel.setArg<cl_uint>(index++, maxTiles);
3238
            force1Kernel.setArg<cl::Buffer>(index++, nb.getBlockCenters().getDeviceBuffer());
3239
            force1Kernel.setArg<cl::Buffer>(index++, nb.getBlockBoundingBoxes().getDeviceBuffer());
3240
            force1Kernel.setArg<cl::Buffer>(index++, nb.getInteractingAtoms().getDeviceBuffer());
3241
        }
3242
3243
        else
            force1Kernel.setArg<cl_uint>(index++, cl.getNumAtomBlocks()*(cl.getNumAtomBlocks()+1)/2);
3244
        force1Kernel.setArg<cl::Buffer>(index++, nb.getExclusionTiles().getDeviceBuffer());
3245
        program = cl.createProgram(OpenCLKernelSources::gbsaObcReductions, defines);
3246
3247
        reduceBornSumKernel = cl::Kernel(program, "reduceBornSum");
        reduceBornSumKernel.setArg<cl_int>(0, cl.getPaddedNumAtoms());
Peter Eastman's avatar
Peter Eastman committed
3248
        reduceBornSumKernel.setArg<cl_int>(1, nb.getNumForceBuffers());
3249
3250
3251
        reduceBornSumKernel.setArg<cl_float>(2, 1.0f);
        reduceBornSumKernel.setArg<cl_float>(3, 0.8f);
        reduceBornSumKernel.setArg<cl_float>(4, 4.85f);
peastman's avatar
peastman committed
3252
3253
3254
3255
        reduceBornSumKernel.setArg<cl::Buffer>(5, (useLong ? longBornSum.getDeviceBuffer() : bornSum.getDeviceBuffer()));
        reduceBornSumKernel.setArg<cl::Buffer>(6, params.getDeviceBuffer());
        reduceBornSumKernel.setArg<cl::Buffer>(7, bornRadii.getDeviceBuffer());
        reduceBornSumKernel.setArg<cl::Buffer>(8, obcChain.getDeviceBuffer());
3256
        reduceBornForceKernel = cl::Kernel(program, "reduceBornForce");
3257
3258
3259
        index = 0;
        reduceBornForceKernel.setArg<cl_int>(index++, cl.getPaddedNumAtoms());
        reduceBornForceKernel.setArg<cl_int>(index++, nb.getNumForceBuffers());
peastman's avatar
peastman committed
3260
        reduceBornForceKernel.setArg<cl::Buffer>(index++, bornForce.getDeviceBuffer());
3261
        if (useLong)
peastman's avatar
peastman committed
3262
            reduceBornForceKernel.setArg<cl::Buffer>(index++, longBornForce.getDeviceBuffer());
3263
        reduceBornForceKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
3264
3265
3266
        reduceBornForceKernel.setArg<cl::Buffer>(index++, params.getDeviceBuffer());
        reduceBornForceKernel.setArg<cl::Buffer>(index++, bornRadii.getDeviceBuffer());
        reduceBornForceKernel.setArg<cl::Buffer>(index++, obcChain.getDeviceBuffer());
3267
    }
3268
    force1Kernel.setArg<cl_int>(6, includeEnergy);
3269
    if (nb.getUseCutoff()) {
3270
3271
        setPeriodicBoxArgs(cl, computeBornSumKernel, 6);
        setPeriodicBoxArgs(cl, force1Kernel, 9);
3272
3273
        if (maxTiles < nb.getInteractingTiles().getSize()) {
            maxTiles = nb.getInteractingTiles().getSize();
Peter Eastman's avatar
Bug fix  
Peter Eastman committed
3274
            computeBornSumKernel.setArg<cl::Buffer>(4, nb.getInteractingTiles().getDeviceBuffer());
3275
3276
3277
3278
3279
            computeBornSumKernel.setArg<cl_uint>(11, maxTiles);
            computeBornSumKernel.setArg<cl::Buffer>(14, nb.getInteractingAtoms().getDeviceBuffer());
            force1Kernel.setArg<cl::Buffer>(7, nb.getInteractingTiles().getDeviceBuffer());
            force1Kernel.setArg<cl_uint>(14, maxTiles);
            force1Kernel.setArg<cl::Buffer>(17, nb.getInteractingAtoms().getDeviceBuffer());
3280
        }
3281
    }
3282
    cl.executeKernel(computeBornSumKernel, nb.getNumForceThreadBlocks()*nb.getForceThreadBlockSize(), nb.getForceThreadBlockSize());
3283
    cl.executeKernel(reduceBornSumKernel, cl.getPaddedNumAtoms());
3284
    cl.executeKernel(force1Kernel, nb.getNumForceThreadBlocks()*nb.getForceThreadBlockSize(), nb.getForceThreadBlockSize());
3285
    cl.executeKernel(reduceBornForceKernel, cl.getPaddedNumAtoms());
3286
    return 0.0;
3287
}
3288

3289
3290
3291
3292
3293
3294
3295
3296
3297
void OpenCLCalcGBSAOBCForceKernel::copyParametersToContext(ContextImpl& context, const GBSAOBCForce& force) {
    // Make sure the new parameters are acceptable.
    
    int numParticles = force.getNumParticles();
    if (numParticles != cl.getNumAtoms())
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
    
    // Record the per-particle parameters.
    
3298
    vector<double> chargeVector(cl.getPaddedNumAtoms(), 0.0);
3299
    vector<mm_float2> paramsVector(cl.getPaddedNumAtoms());
3300
3301
3302
3303
    const double dielectricOffset = 0.009;
    for (int i = 0; i < numParticles; i++) {
        double charge, radius, scalingFactor;
        force.getParticleParameters(i, charge, radius, scalingFactor);
3304
        chargeVector[i] = charge;
3305
3306
3307
        radius -= dielectricOffset;
        paramsVector[i] = mm_float2((float) radius, (float) (scalingFactor*radius));
    }
3308
3309
    for (int i = numParticles; i < cl.getPaddedNumAtoms(); i++)
        paramsVector[i] = mm_float2(1,1);
3310
3311
3312
3313
3314
3315
3316
3317
    if (cl.getUseDoublePrecision())
        charges.upload(chargeVector);
    else {
        vector<float> c(charges.getSize());
        for (int i = 0; i < c.size(); i++)
            c[i] = (float) chargeVector[i];
        charges.upload(c);
    }
peastman's avatar
peastman committed
3318
    params.upload(paramsVector);
3319
3320
3321
    
    // Mark that the current reordering may be invalid.
    
3322
    cl.invalidateMolecules(info);
3323
3324
}

3325
class OpenCLCalcCustomGBForceKernel::ForceInfo : public OpenCLForceInfo {
3326
public:
3327
    ForceInfo(int requiredBuffers, const CustomGBForce& force) : OpenCLForceInfo(requiredBuffers), force(force) {
3328
3329
3330
3331
3332
3333
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        vector<double> params1;
        vector<double> params2;
        force.getParticleParameters(particle1, params1);
        force.getParticleParameters(particle2, params2);
3334
        for (int i = 0; i < (int) params1.size(); i++)
3335
3336
3337
3338
3339
3340
3341
            if (params1[i] != params2[i])
                return false;
        return true;
    }
    int getNumParticleGroups() {
        return force.getNumExclusions();
    }
Peter Eastman's avatar
Peter Eastman committed
3342
    void getParticlesInGroup(int index, vector<int>& particles) {
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
        int particle1, particle2;
        force.getExclusionParticles(index, particle1, particle2);
        particles.resize(2);
        particles[0] = particle1;
        particles[1] = particle2;
    }
    bool areGroupsIdentical(int group1, int group2) {
        return true;
    }
private:
    const CustomGBForce& force;
};

OpenCLCalcCustomGBForceKernel::~OpenCLCalcCustomGBForceKernel() {
    if (params != NULL)
        delete params;
    if (computedValues != NULL)
        delete computedValues;
3361
3362
    if (energyDerivs != NULL)
        delete energyDerivs;
3363
3364
    if (energyDerivChain != NULL)
        delete energyDerivChain;
peastman's avatar
peastman committed
3365
3366
    for (auto d : dValuedParam)
        delete d;
3367
3368
3369
}

void OpenCLCalcCustomGBForceKernel::initialize(const System& system, const CustomGBForce& force) {
3370
3371
    if (cl.getPlatformData().contexts.size() > 1)
        throw OpenMMException("CustomGBForce does not support using multiple OpenCL devices");
3372
    cutoff = force.getCutoffDistance();
3373
    bool useExclusionsForValue = false;
3374
    numComputedValues = force.getNumComputedValues();
3375
3376
    vector<string> computedValueNames(force.getNumComputedValues());
    vector<string> computedValueExpressions(force.getNumComputedValues());
3377
3378
    if (force.getNumComputedValues() > 0) {
        CustomGBForce::ComputationType type;
3379
        force.getComputedValueParameters(0, computedValueNames[0], computedValueExpressions[0], type);
3380
3381
3382
3383
        if (type == CustomGBForce::SingleParticle)
            throw OpenMMException("OpenCLPlatform requires that the first computed value for a CustomGBForce be of type ParticlePair or ParticlePairNoExclusions.");
        useExclusionsForValue = (type == CustomGBForce::ParticlePair);
        for (int i = 1; i < force.getNumComputedValues(); i++) {
3384
            force.getComputedValueParameters(i, computedValueNames[i], computedValueExpressions[i], type);
3385
3386
3387
3388
3389
3390
3391
            if (type != CustomGBForce::SingleParticle)
                throw OpenMMException("OpenCLPlatform requires that a CustomGBForce only have one computed value of type ParticlePair or ParticlePairNoExclusions.");
        }
    }
    int forceIndex;
    for (forceIndex = 0; forceIndex < system.getNumForces() && &system.getForce(forceIndex) != &force; ++forceIndex)
        ;
3392
    string prefix = "custom"+cl.intToString(forceIndex)+"_";
3393
3394
3395
3396

    // Record parameters and exclusions.

    int numParticles = force.getNumParticles();
3397
3398
3399
3400
    int paddedNumParticles = cl.getPaddedNumAtoms();
    int numParams = force.getNumPerParticleParameters();
    params = new OpenCLParameterSet(cl, force.getNumPerParticleParameters(), paddedNumParticles, "customGBParameters", true);
    computedValues = new OpenCLParameterSet(cl, force.getNumComputedValues(), paddedNumParticles, "customGBComputedValues", true, cl.getUseDoublePrecision());
3401
    if (force.getNumGlobalParameters() > 0)
peastman's avatar
peastman committed
3402
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customGBGlobals", CL_MEM_READ_ONLY);
3403
    vector<vector<cl_float> > paramVector(paddedNumParticles, vector<cl_float>(numParams, 0));
3404
3405
3406
3407
    vector<vector<int> > exclusionList(numParticles);
    for (int i = 0; i < numParticles; i++) {
        vector<double> parameters;
        force.getParticleParameters(i, parameters);
3408
        for (int j = 0; j < (int) parameters.size(); j++)
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
            paramVector[i][j] = (cl_float) parameters[j];
        exclusionList[i].push_back(i);
    }
    for (int i = 0; i < force.getNumExclusions(); i++) {
        int particle1, particle2;
        force.getExclusionParticles(i, particle1, particle2);
        exclusionList[particle1].push_back(particle2);
        exclusionList[particle2].push_back(particle1);
    }
    params->setParameterValues(paramVector);

    // Record the tabulated functions.

    map<string, Lepton::CustomFunction*> functions;
    vector<pair<string, string> > functionDefinitions;
3424
    vector<const TabulatedFunction*> functionList;
3425
    stringstream tableArgs;
peastman's avatar
peastman committed
3426
    tabulatedFunctions.resize(force.getNumTabulatedFunctions());
3427
3428
3429
    for (int i = 0; i < force.getNumTabulatedFunctions(); i++) {
        functionList.push_back(&force.getTabulatedFunction(i));
        string name = force.getTabulatedFunctionName(i);
3430
        string arrayName = prefix+"table"+cl.intToString(i);
3431
        functionDefinitions.push_back(make_pair(name, arrayName));
3432
        functions[name] = cl.getExpressionUtilities().getFunctionPlaceholder(force.getTabulatedFunction(i));
peastman's avatar
peastman committed
3433
        int width;
3434
        vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(force.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
3435
3436
3437
        tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
        tabulatedFunctions[i].upload(f);
        cl.getNonbondedUtilities().addArgument(OpenCLNonbondedUtilities::ParameterInfo(arrayName, "float", width, width*sizeof(float), tabulatedFunctions[i].getDeviceBuffer()));
3438
3439
3440
3441
        tableArgs << ", __global const float";
        if (width > 1)
            tableArgs << width;
        tableArgs << "* restrict " << arrayName;
3442
3443
    }

3444
    // Record the global parameters.
3445
3446
3447
3448
3449
3450
3451

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
peastman's avatar
peastman committed
3452
3453
    if (globals.isInitialized())
        globals.upload(globalParamValues);
3454
3455
3456

    // Record derivatives of expressions needed for the chain rule terms.

3457
    vector<vector<Lepton::ParsedExpression> > valueGradientExpressions(force.getNumComputedValues());
3458
    vector<vector<Lepton::ParsedExpression> > valueDerivExpressions(force.getNumComputedValues());
3459
    vector<vector<Lepton::ParsedExpression> > valueParamDerivExpressions(force.getNumComputedValues());
Peter Eastman's avatar
Peter Eastman committed
3460
    needParameterGradient = false;
3461
    for (int i = 0; i < force.getNumComputedValues(); i++) {
3462
        Lepton::ParsedExpression ex = Lepton::Parser::parse(computedValueExpressions[i], functions).optimize();
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
        if (i > 0) {
            valueGradientExpressions[i].push_back(ex.differentiate("x").optimize());
            valueGradientExpressions[i].push_back(ex.differentiate("y").optimize());
            valueGradientExpressions[i].push_back(ex.differentiate("z").optimize());
            if (!isZeroExpression(valueGradientExpressions[i][0]) || !isZeroExpression(valueGradientExpressions[i][1]) || !isZeroExpression(valueGradientExpressions[i][2]))
                needParameterGradient = true;
             for (int j = 0; j < i; j++)
                valueDerivExpressions[i].push_back(ex.differentiate(computedValueNames[j]).optimize());
        }
        for (int j = 0; j < force.getNumEnergyParameterDerivatives(); j++)
            valueParamDerivExpressions[i].push_back(ex.differentiate(force.getEnergyParameterDerivativeName(j)).optimize());
3474
    }
3475
    vector<vector<Lepton::ParsedExpression> > energyDerivExpressions(force.getNumEnergyTerms());
3476
    vector<vector<Lepton::ParsedExpression> > energyParamDerivExpressions(force.getNumEnergyTerms());
Peter Eastman's avatar
Peter Eastman committed
3477
    vector<bool> needChainForValue(force.getNumComputedValues(), false);
3478
3479
3480
3481
3482
3483
    for (int i = 0; i < force.getNumEnergyTerms(); i++) {
        string expression;
        CustomGBForce::ComputationType type;
        force.getEnergyTermParameters(i, expression, type);
        Lepton::ParsedExpression ex = Lepton::Parser::parse(expression, functions).optimize();
        for (int j = 0; j < force.getNumComputedValues(); j++) {
Peter Eastman's avatar
Peter Eastman committed
3484
            if (type == CustomGBForce::SingleParticle) {
3485
                energyDerivExpressions[i].push_back(ex.differentiate(computedValueNames[j]).optimize());
Peter Eastman's avatar
Peter Eastman committed
3486
3487
3488
                if (!isZeroExpression(energyDerivExpressions[i].back()))
                    needChainForValue[j] = true;
            }
3489
3490
            else {
                energyDerivExpressions[i].push_back(ex.differentiate(computedValueNames[j]+"1").optimize());
Peter Eastman's avatar
Peter Eastman committed
3491
3492
                if (!isZeroExpression(energyDerivExpressions[i].back()))
                    needChainForValue[j] = true;
3493
                energyDerivExpressions[i].push_back(ex.differentiate(computedValueNames[j]+"2").optimize());
Peter Eastman's avatar
Peter Eastman committed
3494
3495
                if (!isZeroExpression(energyDerivExpressions[i].back()))
                    needChainForValue[j] = true;
3496
3497
            }
        }
3498
3499
        for (int j = 0; j < force.getNumEnergyParameterDerivatives(); j++)
            energyParamDerivExpressions[i].push_back(ex.differentiate(force.getEnergyParameterDerivativeName(j)).optimize());
3500
    }
3501
    bool deviceIsCpu = (cl.getDevice().getInfo<CL_DEVICE_TYPE>() == CL_DEVICE_TYPE_CPU);
3502
    bool useLong = cl.getSupports64BitGlobalAtomics();
3503
    if (useLong) {
peastman's avatar
peastman committed
3504
        longEnergyDerivs.initialize<cl_long>(cl, force.getNumComputedValues()*cl.getPaddedNumAtoms(), "customGBLongEnergyDerivatives");
Peter Eastman's avatar
Peter Eastman committed
3505
        energyDerivs = new OpenCLParameterSet(cl, force.getNumComputedValues(), cl.getPaddedNumAtoms(), "customGBEnergyDerivatives", true);
3506
3507
    }
    else
Peter Eastman's avatar
Peter Eastman committed
3508
        energyDerivs = new OpenCLParameterSet(cl, force.getNumComputedValues(), cl.getPaddedNumAtoms()*cl.getNonbondedUtilities().getNumForceBuffers(), "customGBEnergyDerivatives", true);
3509
    energyDerivChain = new OpenCLParameterSet(cl, force.getNumComputedValues(), cl.getPaddedNumAtoms(), "customGBEnergyDerivativeChain", true);
3510
3511
    int elementSize = (cl.getUseDoublePrecision() ? sizeof(cl_double) : sizeof(cl_float));
    needEnergyParamDerivs = (force.getNumEnergyParameterDerivatives() > 0);
peastman's avatar
peastman committed
3512
    dValue0dParam.resize(force.getNumEnergyParameterDerivatives());
3513
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
3514
        dValuedParam.push_back(new OpenCLParameterSet(cl, force.getNumComputedValues(), cl.getPaddedNumAtoms(), "dValuedParam", true, cl.getUseDoublePrecision()));
3515
        if (useLong)
peastman's avatar
peastman committed
3516
            dValue0dParam[i].initialize<cl_long>(cl, cl.getPaddedNumAtoms(), "dValue0dParam");
3517
        else
peastman's avatar
peastman committed
3518
3519
            dValue0dParam[i].initialize(cl, cl.getPaddedNumAtoms()*cl.getNonbondedUtilities().getNumForceBuffers(), elementSize, "dValue0dParam");
        cl.addAutoclearBuffer(dValue0dParam[i]);
3520
3521
3522
        string name = force.getEnergyParameterDerivativeName(i);
        cl.addEnergyParameterDerivative(name);
    }
3523

3524
3525
    // Create the kernels.

3526
3527
    bool useCutoff = (force.getNonbondedMethod() != CustomGBForce::NoCutoff);
    bool usePeriodic = (force.getNonbondedMethod() != CustomGBForce::NoCutoff && force.getNonbondedMethod() != CustomGBForce::CutoffNonPeriodic);
3528
3529
3530
    {
        // Create the N2 value kernel.

3531
        vector<pair<ExpressionTreeNode, string> > variables;
3532
        map<string, string> rename;
3533
3534
3535
3536
        ExpressionTreeNode rnode(new Operation::Variable("r"));
        variables.push_back(make_pair(rnode, "r"));
        variables.push_back(make_pair(ExpressionTreeNode(new Operation::Square(), rnode), "r2"));
        variables.push_back(make_pair(ExpressionTreeNode(new Operation::Reciprocal(), rnode), "invR"));
3537
3538
        for (int i = 0; i < force.getNumPerParticleParameters(); i++) {
            const string& name = force.getPerParticleParameterName(i);
3539
3540
            variables.push_back(makeVariable(name+"1", "params"+params->getParameterSuffix(i, "1")));
            variables.push_back(makeVariable(name+"2", "params"+params->getParameterSuffix(i, "2")));
3541
3542
            rename[name+"1"] = name+"2";
            rename[name+"2"] = name+"1";
3543
3544
3545
        }
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
3546
            string value = "globals["+cl.intToString(i)+"]";
3547
            variables.push_back(makeVariable(name, value));
3548
        }
3549
3550
        map<string, Lepton::ParsedExpression> n2ValueExpressions;
        stringstream n2ValueSource;
3551
3552
3553
        Lepton::ParsedExpression ex = Lepton::Parser::parse(computedValueExpressions[0], functions).optimize();
        n2ValueExpressions["tempValue1 = "] = ex;
        n2ValueExpressions["tempValue2 = "] = ex.renameVariables(rename);
3554
3555
3556
3557
3558
3559
3560
        for (int i = 0; i < valueParamDerivExpressions[0].size(); i++) {
            string variableBase = "temp_dValue0dParam"+cl.intToString(i+1);
            if (!isZeroExpression(valueParamDerivExpressions[0][i])) {
                n2ValueExpressions[variableBase+"_1 = "] = valueParamDerivExpressions[0][i];
                n2ValueExpressions[variableBase+"_2 = "] = valueParamDerivExpressions[0][i].renameVariables(rename);
            }
        }
3561
        n2ValueSource << cl.getExpressionUtilities().createExpressions(n2ValueExpressions, variables, functionList, functionDefinitions, "temp");
3562
        map<string, string> replacements;
Peter Eastman's avatar
Peter Eastman committed
3563
3564
        string n2ValueStr = n2ValueSource.str();
        replacements["COMPUTE_VALUE"] = n2ValueStr;
3565
        stringstream extraArgs, loadLocal1, loadLocal2, load1, load2, tempDerivs1, tempDerivs2, storeDeriv1, storeDeriv2;
3566
        if (force.getNumGlobalParameters() > 0)
Peter Eastman's avatar
Peter Eastman committed
3567
            extraArgs << ", __global const float* globals";
Peter Eastman's avatar
Peter Eastman committed
3568
        pairValueUsesParam.resize(params->getBuffers().size(), false);
3569
3570
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
3571
            string paramName = "params"+cl.intToString(i+1);
Peter Eastman's avatar
Peter Eastman committed
3572
3573
3574
3575
3576
3577
3578
3579
            if (n2ValueStr.find(paramName+"1") != n2ValueStr.npos || n2ValueStr.find(paramName+"2") != n2ValueStr.npos) {
                extraArgs << ", __global const " << buffer.getType() << "* restrict global_" << paramName << ", __local " << buffer.getType() << "* restrict local_" << paramName;
                loadLocal1 << "local_" << paramName << "[localAtomIndex] = " << paramName << "1;\n";
                loadLocal2 << "local_" << paramName << "[localAtomIndex] = global_" << paramName << "[j];\n";
                load1 << buffer.getType() << " " << paramName << "1 = global_" << paramName << "[atom1];\n";
                load2 << buffer.getType() << " " << paramName << "2 = local_" << paramName << "[atom2];\n";
                pairValueUsesParam[i] = true;
            }
3580
        }
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
        for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
            string derivName = "dValue0dParam"+cl.intToString(i+1);
            if (useLong)
                extraArgs << ", __global long* restrict global_" << derivName;
            else
                extraArgs << ", __global real* restrict global_" << derivName;
            extraArgs << ", __local real* restrict local_" << derivName;
            loadLocal2 << "local_" << derivName << "[localAtomIndex] = 0;\n";
            load1 << "real " << derivName << " = 0;\n";
            if (!isZeroExpression(valueParamDerivExpressions[0][i])) {
                load2 << "real temp_" << derivName << "_1 = 0;\n";
                load2 << "real temp_" << derivName << "_2 = 0;\n";
                tempDerivs1 << derivName << " += temp_" << derivName << "_1;\n";
peastman's avatar
peastman committed
3594
3595
3596
3597
                if (deviceIsCpu)
                    tempDerivs2 << "local_" << derivName << "[j] += temp_" << derivName << "_2;\n";
                else
                    tempDerivs2 << "local_" << derivName << "[tbx+tj] += temp_" << derivName << "_2;\n";
3598
3599
                if (useLong) {
                    storeDeriv1 << "atom_add(&global_" << derivName << "[offset1], (long) (" << derivName << "*0x100000000));\n";
peastman's avatar
peastman committed
3600
3601
3602
3603
                    if (deviceIsCpu)
                        storeDeriv2 << "atom_add(&global_" << derivName << "[offset2], (long) (local_" << derivName << "[tgx]*0x100000000));\n";
                    else
                        storeDeriv2 << "atom_add(&global_" << derivName << "[offset2], (long) (local_" << derivName << "[get_local_id(0)]*0x100000000));\n";
3604
3605
3606
                }
                else {
                    storeDeriv1 << "global_" << derivName << "[offset1] += " << derivName << ";\n";
peastman's avatar
peastman committed
3607
3608
3609
3610
                    if (deviceIsCpu)
                        storeDeriv2 << "global_" << derivName << "[offset2] += local_" << derivName << "[tgx];\n";
                    else
                        storeDeriv2 << "global_" << derivName << "[offset2] += local_" << derivName << "[get_local_id(0)];\n";
3611
3612
3613
                }
            }
        }
3614
        replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
3615
3616
3617
3618
        replacements["LOAD_LOCAL_PARAMETERS_FROM_1"] = loadLocal1.str();
        replacements["LOAD_LOCAL_PARAMETERS_FROM_GLOBAL"] = loadLocal2.str();
        replacements["LOAD_ATOM1_PARAMETERS"] = load1.str();
        replacements["LOAD_ATOM2_PARAMETERS"] = load2.str();
3619
3620
3621
3622
        replacements["ADD_TEMP_DERIVS1"] = tempDerivs1.str();
        replacements["ADD_TEMP_DERIVS2"] = tempDerivs2.str();
        replacements["STORE_PARAM_DERIVS1"] = storeDeriv1.str();
        replacements["STORE_PARAM_DERIVS2"] = storeDeriv2.str();
3623
        if (useCutoff)
3624
            pairValueDefines["USE_CUTOFF"] = "1";
3625
        if (usePeriodic)
3626
            pairValueDefines["USE_PERIODIC"] = "1";
3627
        if (useExclusionsForValue)
3628
3629
            pairValueDefines["USE_EXCLUSIONS"] = "1";
        pairValueDefines["FORCE_WORK_GROUP_SIZE"] = cl.intToString(cl.getNonbondedUtilities().getForceThreadBlockSize());
3630
        pairValueDefines["CUTOFF_SQUARED"] = cl.doubleToString(cutoff*cutoff);
3631
3632
3633
3634
        pairValueDefines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
        pairValueDefines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
        pairValueDefines["NUM_BLOCKS"] = cl.intToString(cl.getNumAtomBlocks());
        pairValueDefines["TILE_SIZE"] = cl.intToString(OpenCLContext::TileSize);
3635
3636
3637
3638
        string file;
        if (deviceIsCpu)
            file = OpenCLKernelSources::customGBValueN2_cpu;
        else
3639
3640
            file = OpenCLKernelSources::customGBValueN2;
        pairValueSrc = cl.replaceStrings(file, replacements);
3641
3642
        if (useExclusionsForValue)
            cl.getNonbondedUtilities().requestExclusions(exclusionList);
3643
3644
3645
3646
    }
    {
        // Create the kernel to reduce the N2 value and calculate other values.

3647
        stringstream reductionSource, extraArgs, deriv0;
3648
        if (force.getNumGlobalParameters() > 0)
Peter Eastman's avatar
Peter Eastman committed
3649
            extraArgs << ", __global const float* globals";
3650
3651
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
3652
            string paramName = "params"+cl.intToString(i+1);
3653
            extraArgs << ", __global const " << buffer.getType() << "* restrict " << paramName;
3654
3655
3656
        }
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = computedValues->getBuffers()[i];
3657
            string valueName = "values"+cl.intToString(i+1);
3658
            extraArgs << ", __global " << buffer.getType() << "* restrict global_" << valueName;
3659
3660
            reductionSource << buffer.getType() << " local_" << valueName << ";\n";
        }
3661
3662
3663
3664
        for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
            string variableName = "dValuedParam_0_"+cl.intToString(i);
            if (useLong) {
                extraArgs << ", __global const long* restrict dValue0dParam" << i;
3665
                deriv0 << "real " << variableName << " = (1.0f/0x100000000)*dValue0dParam" << i << "[index];\n";
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
            }
            else {
                extraArgs << ", __global const real* restrict dValue0dParam" << i;
                deriv0 << "real " << variableName << " = dValue0dParam" << i << "[index];\n";
                deriv0 << "for (int i = index+bufferSize; i < totalSize; i += bufferSize)\n";
                deriv0 << "    " << variableName << " += dValue0dParam" << i << "[i];\n";
            }
            for (int j = 0; j < dValuedParam[i]->getBuffers().size(); j++)
                extraArgs << ", __global real* restrict global_dValuedParam_" << j << "_" << i;
            deriv0 << "global_dValuedParam_0_" << i << "[index] = dValuedParam_0_" << i << ";\n";
        }
3677
        reductionSource << "local_values" << computedValues->getParameterSuffix(0) << " = sum;\n";
3678
        map<string, string> variables;
3679
3680
3681
        variables["x"] = "pos.x";
        variables["y"] = "pos.y";
        variables["z"] = "pos.z";
3682
3683
3684
        for (int i = 0; i < force.getNumPerParticleParameters(); i++)
            variables[force.getPerParticleParameterName(i)] = "params"+params->getParameterSuffix(i, "[index]");
        for (int i = 0; i < force.getNumGlobalParameters(); i++)
3685
            variables[force.getGlobalParameterName(i)] = "globals["+cl.intToString(i)+"]";
3686
3687
3688
3689
        for (int i = 1; i < force.getNumComputedValues(); i++) {
            variables[computedValueNames[i-1]] = "local_values"+computedValues->getParameterSuffix(i-1);
            map<string, Lepton::ParsedExpression> valueExpressions;
            valueExpressions["local_values"+computedValues->getParameterSuffix(i)+" = "] = Lepton::Parser::parse(computedValueExpressions[i], functions).optimize();
3690
            reductionSource << cl.getExpressionUtilities().createExpressions(valueExpressions, variables, functionList, functionDefinitions, "value"+cl.intToString(i)+"_temp");
3691
        }
3692
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
3693
            string valueName = "values"+cl.intToString(i+1);
3694
3695
            reductionSource << "global_" << valueName << "[index] = local_" << valueName << ";\n";
        }
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
        if (needEnergyParamDerivs) {
            map<string, Lepton::ParsedExpression> derivExpressions;
            for (int i = 1; i < force.getNumComputedValues(); i++) {
                for (int j = 0; j < valueParamDerivExpressions[i].size(); j++)
                    derivExpressions["real dValuedParam_"+cl.intToString(i)+"_"+cl.intToString(j)+" = "] = valueParamDerivExpressions[i][j];
                for (int j = 0; j < i; j++)
                    derivExpressions["real dVdV_"+cl.intToString(i)+"_"+cl.intToString(j)+" = "] = valueDerivExpressions[i][j];
            }
            reductionSource << cl.getExpressionUtilities().createExpressions(derivExpressions, variables, functionList, functionDefinitions, "derivChain_temp");
            for (int i = 1; i < force.getNumComputedValues(); i++) {
                for (int j = 0; j < i; j++)
                    for (int k = 0; k < valueParamDerivExpressions[i].size(); k++)
                        reductionSource << "dValuedParam_" << i << "_" << k << " += dVdV_" << i << "_" << j << "*dValuedParam_" << j <<"_" << k << ";\n";
                for (int j = 0; j < valueParamDerivExpressions[i].size(); j++)
                    reductionSource << "global_dValuedParam_" << i << "_" << j << "[index] = dValuedParam_" << i << "_" << j << ";\n";
            }
        }
3713
        map<string, string> replacements;
3714
        replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
3715
        replacements["REDUCE_PARAM0_DERIV"] = deriv0.str();
3716
3717
        replacements["COMPUTE_VALUES"] = reductionSource.str();
        map<string, string> defines;
3718
        defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
3719
        cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customGBValuePerParticle, replacements), defines);
3720
3721
3722
3723
3724
        perParticleValueKernel = cl::Kernel(program, "computePerParticleValues");
    }
    {
        // Create the N2 energy kernel.

3725
3726
3727
3728
3729
        vector<pair<ExpressionTreeNode, string> > variables;
        ExpressionTreeNode rnode(new Operation::Variable("r"));
        variables.push_back(make_pair(rnode, "r"));
        variables.push_back(make_pair(ExpressionTreeNode(new Operation::Square(), rnode), "r2"));
        variables.push_back(make_pair(ExpressionTreeNode(new Operation::Reciprocal(), rnode), "invR"));
3730
3731
        for (int i = 0; i < force.getNumPerParticleParameters(); i++) {
            const string& name = force.getPerParticleParameterName(i);
3732
3733
            variables.push_back(makeVariable(name+"1", "params"+params->getParameterSuffix(i, "1")));
            variables.push_back(makeVariable(name+"2", "params"+params->getParameterSuffix(i, "2")));
3734
3735
        }
        for (int i = 0; i < force.getNumComputedValues(); i++) {
3736
3737
            variables.push_back(makeVariable(computedValueNames[i]+"1", "values"+computedValues->getParameterSuffix(i, "1")));
            variables.push_back(makeVariable(computedValueNames[i]+"2", "values"+computedValues->getParameterSuffix(i, "2")));
3738
3739
        }
        for (int i = 0; i < force.getNumGlobalParameters(); i++)
3740
            variables.push_back(makeVariable(force.getGlobalParameterName(i), "globals["+cl.intToString(i)+"]"));
3741
        stringstream n2EnergySource;
3742
        bool anyExclusions = (force.getNumExclusions() > 0);
3743
3744
3745
3746
3747
3748
        for (int i = 0; i < force.getNumEnergyTerms(); i++) {
            string expression;
            CustomGBForce::ComputationType type;
            force.getEnergyTermParameters(i, expression, type);
            if (type == CustomGBForce::SingleParticle)
                continue;
3749
            bool exclude = (anyExclusions && type == CustomGBForce::ParticlePair);
3750
            map<string, Lepton::ParsedExpression> n2EnergyExpressions;
3751
3752
            n2EnergyExpressions["tempEnergy += "] = Lepton::Parser::parse(expression, functions).optimize();
            n2EnergyExpressions["dEdR += "] = Lepton::Parser::parse(expression, functions).differentiate("r").optimize();
3753
3754
            if (useLong) {
                for (int j = 0; j < force.getNumComputedValues(); j++) {
Peter Eastman's avatar
Peter Eastman committed
3755
                    if (needChainForValue[j]) {
3756
3757
3758
                        string index = cl.intToString(j+1);
                        n2EnergyExpressions["/*"+cl.intToString(i+1)+"*/ deriv"+index+"_1 += "] = energyDerivExpressions[i][2*j];
                        n2EnergyExpressions["/*"+cl.intToString(i+1)+"*/ deriv"+index+"_2 += "] = energyDerivExpressions[i][2*j+1];
Peter Eastman's avatar
Peter Eastman committed
3759
                    }
3760
3761
3762
3763
                }
            }
            else {
                for (int j = 0; j < force.getNumComputedValues(); j++) {
Peter Eastman's avatar
Peter Eastman committed
3764
                    if (needChainForValue[j]) {
3765
3766
                        n2EnergyExpressions["/*"+cl.intToString(i+1)+"*/ deriv"+energyDerivs->getParameterSuffix(j, "_1")+" += "] = energyDerivExpressions[i][2*j];
                        n2EnergyExpressions["/*"+cl.intToString(i+1)+"*/ deriv"+energyDerivs->getParameterSuffix(j, "_2")+" += "] = energyDerivExpressions[i][2*j+1];
Peter Eastman's avatar
Peter Eastman committed
3767
                    }
3768
                }
3769
            }
3770
3771
            for (int j = 0; j < force.getNumEnergyParameterDerivatives(); j++)
                n2EnergyExpressions["energyParamDeriv"+cl.intToString(j)+" += interactionScale*"] = energyParamDerivExpressions[i][j];
3772
3773
            if (exclude)
                n2EnergySource << "if (!isExcluded) {\n";
3774
            n2EnergySource << cl.getExpressionUtilities().createExpressions(n2EnergyExpressions, variables, functionList, functionDefinitions, "temp");
3775
3776
            if (exclude)
                n2EnergySource << "}\n";
3777
3778
        }
        map<string, string> replacements;
Peter Eastman's avatar
Peter Eastman committed
3779
3780
        string n2EnergyStr = n2EnergySource.str();
        replacements["COMPUTE_INTERACTION"] = n2EnergyStr;
3781
        stringstream extraArgs, loadLocal1, loadLocal2, clearLocal, load1, load2, declare1, recordDeriv, storeDerivs1, storeDerivs2, declareTemps, setTemps, initParamDerivs, saveParamDerivs;
3782
        if (force.getNumGlobalParameters() > 0)
Peter Eastman's avatar
Peter Eastman committed
3783
            extraArgs << ", __global const float* globals";
Peter Eastman's avatar
Peter Eastman committed
3784
        pairEnergyUsesParam.resize(params->getBuffers().size(), false);
3785
3786
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
3787
            string paramName = "params"+cl.intToString(i+1);
Peter Eastman's avatar
Peter Eastman committed
3788
3789
3790
3791
3792
3793
3794
3795
            if (n2EnergyStr.find(paramName+"1") != n2EnergyStr.npos || n2EnergyStr.find(paramName+"2") != n2EnergyStr.npos) {
                extraArgs << ", __global const " << buffer.getType() << "* restrict global_" << paramName << ", __local " << buffer.getType() << "* restrict local_" << paramName;
                loadLocal1 << "local_" << paramName << "[localAtomIndex] = " << paramName << "1;\n";
                loadLocal2 << "local_" << paramName << "[localAtomIndex] = global_" << paramName << "[j];\n";
                load1 << buffer.getType() << " " << paramName << "1 = global_" << paramName << "[atom1];\n";
                load2 << buffer.getType() << " " << paramName << "2 = local_" << paramName << "[atom2];\n";
                pairEnergyUsesParam[i] = true;
            }
3796
        }
Peter Eastman's avatar
Peter Eastman committed
3797
        pairEnergyUsesValue.resize(computedValues->getBuffers().size(), false);
3798
3799
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = computedValues->getBuffers()[i];
3800
            string valueName = "values"+cl.intToString(i+1);
Peter Eastman's avatar
Peter Eastman committed
3801
3802
3803
3804
3805
3806
3807
3808
            if (n2EnergyStr.find(valueName+"1") != n2EnergyStr.npos || n2EnergyStr.find(valueName+"2") != n2EnergyStr.npos) {
                extraArgs << ", __global const " << buffer.getType() << "* restrict global_" << valueName << ", __local " << buffer.getType() << "* restrict local_" << valueName;
                loadLocal1 << "local_" << valueName << "[localAtomIndex] = " << valueName << "1;\n";
                loadLocal2 << "local_" << valueName << "[localAtomIndex] = global_" << valueName << "[j];\n";
                load1 << buffer.getType() << " " << valueName << "1 = global_" << valueName << "[atom1];\n";
                load2 << buffer.getType() << " " << valueName << "2 = local_" << valueName << "[atom2];\n";
                pairEnergyUsesValue[i] = true;
            }
3809
        }
3810
        if (useLong) {
3811
            extraArgs << ", __global long* restrict derivBuffers";
3812
            for (int i = 0; i < force.getNumComputedValues(); i++) {
3813
                string index = cl.intToString(i+1);
3814
                extraArgs << ", __local real* restrict local_deriv" << index;
3815
                clearLocal << "local_deriv" << index << "[localAtomIndex] = 0.0f;\n";
3816
3817
                declare1 << "real deriv" << index << "_1 = 0;\n";
                load2 << "real deriv" << index << "_2 = 0;\n";
3818
3819
3820
                recordDeriv << "local_deriv" << index << "[atom2] += deriv" << index << "_2;\n";
                storeDerivs1 << "STORE_DERIVATIVE_1(" << index << ")\n";
                storeDerivs2 << "STORE_DERIVATIVE_2(" << index << ")\n";
3821
                declareTemps << "__local real tempDerivBuffer" << index << "[64];\n";
3822
3823
3824
3825
3826
3827
                setTemps << "tempDerivBuffer" << index << "[get_local_id(0)] = deriv" << index << "_1;\n";
            }
        }
        else {
            for (int i = 0; i < (int) energyDerivs->getBuffers().size(); i++) {
                const OpenCLNonbondedUtilities::ParameterInfo& buffer = energyDerivs->getBuffers()[i];
3828
                string index = cl.intToString(i+1);
3829
                extraArgs << ", __global " << buffer.getType() << "* restrict derivBuffers" << index << ", __local " << buffer.getType() << "* restrict local_deriv" << index;
3830
3831
3832
3833
3834
3835
3836
3837
3838
                clearLocal << "local_deriv" << index << "[localAtomIndex] = 0.0f;\n";
                declare1 << buffer.getType() << " deriv" << index << "_1 = 0.0f;\n";
                load2 << buffer.getType() << " deriv" << index << "_2 = 0.0f;\n";
                recordDeriv << "local_deriv" << index << "[atom2] += deriv" << index << "_2;\n";
                storeDerivs1 << "STORE_DERIVATIVE_1(" << index << ")\n";
                storeDerivs2 << "STORE_DERIVATIVE_2(" << index << ")\n";
                declareTemps << "__local " << buffer.getType() << " tempDerivBuffer" << index << "[64];\n";
                setTemps << "tempDerivBuffer" << index << "[get_local_id(0)] = deriv" << index << "_1;\n";
            }
3839
        }
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
        if (needEnergyParamDerivs) {
            extraArgs << ", __global mixed* restrict energyParamDerivs";
            const vector<string>& allParamDerivNames = cl.getEnergyParamDerivNames();
            int numDerivs = allParamDerivNames.size();
            for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
                initParamDerivs << "mixed energyParamDeriv" << i << " = 0;\n";
                for (int index = 0; index < numDerivs; index++)
                    if (allParamDerivNames[index] == force.getEnergyParameterDerivativeName(i))
                        saveParamDerivs << "energyParamDerivs[get_global_id(0)*" << numDerivs << "+" << index << "] += energyParamDeriv" << i << ";\n";
            }
        }
3851
3852
3853
        replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
        replacements["LOAD_LOCAL_PARAMETERS_FROM_1"] = loadLocal1.str();
        replacements["LOAD_LOCAL_PARAMETERS_FROM_GLOBAL"] = loadLocal2.str();
3854
        replacements["CLEAR_LOCAL_DERIVATIVES"] = clearLocal.str();
3855
3856
        replacements["LOAD_ATOM1_PARAMETERS"] = load1.str();
        replacements["LOAD_ATOM2_PARAMETERS"] = load2.str();
3857
        replacements["DECLARE_ATOM1_DERIVATIVES"] = declare1.str();
3858
3859
3860
        replacements["RECORD_DERIVATIVE_2"] = recordDeriv.str();
        replacements["STORE_DERIVATIVES_1"] = storeDerivs1.str();
        replacements["STORE_DERIVATIVES_2"] = storeDerivs2.str();
3861
3862
        replacements["DECLARE_TEMP_BUFFERS"] = declareTemps.str();
        replacements["SET_TEMP_BUFFERS"] = setTemps.str();
3863
3864
        replacements["INIT_PARAM_DERIVS"] = initParamDerivs.str();
        replacements["SAVE_PARAM_DERIVS"] = saveParamDerivs.str();
3865
        if (useCutoff)
3866
            pairEnergyDefines["USE_CUTOFF"] = "1";
3867
        if (usePeriodic)
3868
            pairEnergyDefines["USE_PERIODIC"] = "1";
3869
        if (anyExclusions)
3870
3871
            pairEnergyDefines["USE_EXCLUSIONS"] = "1";
        pairEnergyDefines["FORCE_WORK_GROUP_SIZE"] = cl.intToString(cl.getNonbondedUtilities().getForceThreadBlockSize());
3872
        pairEnergyDefines["CUTOFF_SQUARED"] = cl.doubleToString(cutoff*cutoff);
3873
3874
3875
3876
        pairEnergyDefines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
        pairEnergyDefines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
        pairEnergyDefines["NUM_BLOCKS"] = cl.intToString(cl.getNumAtomBlocks());
        pairEnergyDefines["TILE_SIZE"] = cl.intToString(OpenCLContext::TileSize);
3877
3878
3879
3880
        string file;
        if (deviceIsCpu)
            file = OpenCLKernelSources::customGBEnergyN2_cpu;
        else
3881
3882
            file = OpenCLKernelSources::customGBEnergyN2;
        pairEnergySrc = cl.replaceStrings(file, replacements);
3883
3884
3885
3886
    }
    {
        // Create the kernel to reduce the derivatives and calculate per-particle energy terms.

3887
        stringstream compute, extraArgs, reduce, initParamDerivs, saveParamDerivs;
3888
        if (force.getNumGlobalParameters() > 0)
Peter Eastman's avatar
Peter Eastman committed
3889
            extraArgs << ", __global const float* globals";
3890
3891
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
3892
            string paramName = "params"+cl.intToString(i+1);
3893
            extraArgs << ", __global const " << buffer.getType() << "* restrict " << paramName;
3894
3895
3896
        }
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = computedValues->getBuffers()[i];
3897
            string valueName = "values"+cl.intToString(i+1);
3898
            extraArgs << ", __global const " << buffer.getType() << "* restrict " << valueName;
3899
        }
3900
3901
        for (int i = 0; i < (int) energyDerivs->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = energyDerivs->getBuffers()[i];
3902
            string index = cl.intToString(i+1);
3903
            extraArgs << ", __global " << buffer.getType() << "* restrict derivBuffers" << index;
3904
3905
            compute << buffer.getType() << " deriv" << index << " = derivBuffers" << index << "[index];\n";
        }
3906
3907
3908
3909
3910
        for (int i = 0; i < (int) energyDerivChain->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = energyDerivChain->getBuffers()[i];
            string index = cl.intToString(i+1);
            extraArgs << ", __global " << buffer.getType() << "* restrict derivChain" << index;
        }
3911
        if (useLong) {
3912
            extraArgs << ", __global const long* restrict derivBuffersIn";
3913
3914
            for (int i = 0; i < energyDerivs->getNumParameters(); ++i)
                reduce << "derivBuffers" << energyDerivs->getParameterSuffix(i, "[index]") <<
3915
                        " = (1.0f/0x100000000)*derivBuffersIn[index+PADDED_NUM_ATOMS*" << cl.intToString(i) << "];\n";
3916
3917
3918
        }
        else {
            for (int i = 0; i < (int) energyDerivs->getBuffers().size(); i++)
3919
                reduce << "REDUCE_VALUE(derivBuffers" << cl.intToString(i+1) << ", " << energyDerivs->getBuffers()[i].getType() << ")\n";
3920
        }
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
        if (needEnergyParamDerivs) {
            extraArgs << ", __global mixed* restrict energyParamDerivs";
            const vector<string>& allParamDerivNames = cl.getEnergyParamDerivNames();
            int numDerivs = allParamDerivNames.size();
            for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
                initParamDerivs << "mixed energyParamDeriv" << i << " = 0;\n";
                for (int index = 0; index < numDerivs; index++)
                    if (allParamDerivNames[index] == force.getEnergyParameterDerivativeName(i))
                        saveParamDerivs << "energyParamDerivs[get_global_id(0)*" << numDerivs << "+" << index << "] += energyParamDeriv" << i << ";\n";
            }
        }
Peter Eastman's avatar
Peter Eastman committed
3932
3933
3934
        
        // Compute the various expressions.
        
3935
        map<string, string> variables;
3936
3937
3938
        variables["x"] = "pos.x";
        variables["y"] = "pos.y";
        variables["z"] = "pos.z";
3939
3940
3941
        for (int i = 0; i < force.getNumPerParticleParameters(); i++)
            variables[force.getPerParticleParameterName(i)] = "params"+params->getParameterSuffix(i, "[index]");
        for (int i = 0; i < force.getNumGlobalParameters(); i++)
3942
            variables[force.getGlobalParameterName(i)] = "globals["+cl.intToString(i)+"]";
3943
3944
        for (int i = 0; i < force.getNumComputedValues(); i++)
            variables[computedValueNames[i]] = "values"+computedValues->getParameterSuffix(i, "[index]");
Peter Eastman's avatar
Peter Eastman committed
3945
        map<string, Lepton::ParsedExpression> expressions;
3946
3947
3948
3949
3950
3951
        for (int i = 0; i < force.getNumEnergyTerms(); i++) {
            string expression;
            CustomGBForce::ComputationType type;
            force.getEnergyTermParameters(i, expression, type);
            if (type != CustomGBForce::SingleParticle)
                continue;
3952
            Lepton::ParsedExpression parsed = Lepton::Parser::parse(expression, functions).optimize();
3953
            expressions["/*"+cl.intToString(i+1)+"*/ energy += "] = parsed;
3954
            for (int j = 0; j < force.getNumComputedValues(); j++)
3955
                expressions["/*"+cl.intToString(i+1)+"*/ deriv"+energyDerivs->getParameterSuffix(j)+" += "] = energyDerivExpressions[i][j];
3956
3957
3958
3959
            Lepton::ParsedExpression gradx = parsed.differentiate("x").optimize();
            Lepton::ParsedExpression grady = parsed.differentiate("y").optimize();
            Lepton::ParsedExpression gradz = parsed.differentiate("z").optimize();
            if (!isZeroExpression(gradx))
3960
                expressions["/*"+cl.intToString(i+1)+"*/ force.x -= "] = gradx;
3961
            if (!isZeroExpression(grady))
3962
                expressions["/*"+cl.intToString(i+1)+"*/ force.y -= "] = grady;
3963
            if (!isZeroExpression(gradz))
3964
                expressions["/*"+cl.intToString(i+1)+"*/ force.z -= "] = gradz;
3965
3966
            for (int j = 0; j < force.getNumEnergyParameterDerivatives(); j++)
                expressions["/*"+cl.intToString(i+1)+"*/ energyParamDeriv"+cl.intToString(j)+" += "] = energyParamDerivExpressions[i][j];
Peter Eastman's avatar
Peter Eastman committed
3967
3968
3969
        }
        for (int i = 1; i < force.getNumComputedValues(); i++)
            for (int j = 0; j < i; j++)
3970
                expressions["real dV"+cl.intToString(i)+"dV"+cl.intToString(j)+" = "] = valueDerivExpressions[i][j];
3971
        compute << cl.getExpressionUtilities().createExpressions(expressions, variables, functionList, functionDefinitions, "temp");
Peter Eastman's avatar
Peter Eastman committed
3972
3973
3974
        
        // Record values.
        
3975
3976
3977
3978
        for (int i = 0; i < (int) energyDerivs->getBuffers().size(); i++) {
            string index = cl.intToString(i+1);
            compute << "derivBuffers" << index << "[index] = deriv" << index << ";\n";
        }
Peter Eastman's avatar
Peter Eastman committed
3979
3980
        compute << "forceBuffers[index] = forceBuffers[index]+force;\n";
        for (int i = 1; i < force.getNumComputedValues(); i++) {
3981
            compute << "real totalDeriv"<<i<<" = dV"<<i<<"dV0";
Peter Eastman's avatar
Peter Eastman committed
3982
3983
3984
3985
            for (int j = 1; j < i; j++)
                compute << " + totalDeriv"<<j<<"*dV"<<i<<"dV"<<j;
            compute << ";\n";
            compute << "deriv"<<(i+1)<<" *= totalDeriv"<<i<<";\n";
3986
3987
        }
        for (int i = 0; i < (int) energyDerivs->getBuffers().size(); i++) {
3988
            string index = cl.intToString(i+1);
3989
            compute << "derivChain" << index << "[index] = deriv" << index << ";\n";
3990
3991
3992
        }
        map<string, string> replacements;
        replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
3993
3994
        replacements["REDUCE_DERIVATIVES"] = reduce.str();
        replacements["COMPUTE_ENERGY"] = compute.str();
3995
3996
        replacements["INIT_PARAM_DERIVS"] = initParamDerivs.str();
        replacements["SAVE_PARAM_DERIVS"] = saveParamDerivs.str();
3997
        map<string, string> defines;
3998
3999
        defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
        defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
4000
        cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customGBEnergyPerParticle, replacements), defines);
4001
        perParticleEnergyKernel = cl::Kernel(program, "computePerParticleEnergy");
4002
    }
4003
4004
4005
    if (needParameterGradient || needEnergyParamDerivs) {
        // Create the kernel to compute chain rule terms for computed values that depend explicitly on particle coordinates, and for
        // derivatives with respect to global parameters.
Peter Eastman's avatar
Peter Eastman committed
4006

4007
        stringstream compute, extraArgs, initParamDerivs, saveParamDerivs;
Peter Eastman's avatar
Peter Eastman committed
4008
        if (force.getNumGlobalParameters() > 0)
Peter Eastman's avatar
Peter Eastman committed
4009
            extraArgs << ", __global const float* globals";
Peter Eastman's avatar
Peter Eastman committed
4010
4011
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
4012
            string paramName = "params"+cl.intToString(i+1);
4013
            extraArgs << ", __global const " << buffer.getType() << "* restrict " << paramName;
Peter Eastman's avatar
Peter Eastman committed
4014
4015
4016
        }
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = computedValues->getBuffers()[i];
4017
            string valueName = "values"+cl.intToString(i+1);
4018
            extraArgs << ", __global const " << buffer.getType() << "* restrict " << valueName;
Peter Eastman's avatar
Peter Eastman committed
4019
4020
4021
        }
        for (int i = 0; i < (int) energyDerivs->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = energyDerivs->getBuffers()[i];
4022
            string index = cl.intToString(i+1);
4023
            extraArgs << ", __global " << buffer.getType() << "* restrict derivBuffers" << index;
Peter Eastman's avatar
Peter Eastman committed
4024
4025
            compute << buffer.getType() << " deriv" << index << " = derivBuffers" << index << "[index];\n";
        }
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
        if (needEnergyParamDerivs) {
            extraArgs << ", __global mixed* restrict energyParamDerivs";
            const vector<string>& allParamDerivNames = cl.getEnergyParamDerivNames();
            int numDerivs = allParamDerivNames.size();
            for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
                for (int j = 0; j < dValuedParam[i]->getBuffers().size(); j++)
                    extraArgs << ", __global real* restrict dValuedParam_" << j << "_" << i;
                initParamDerivs << "mixed energyParamDeriv" << i << " = 0;\n";
                for (int index = 0; index < numDerivs; index++)
                    if (allParamDerivNames[index] == force.getEnergyParameterDerivativeName(i))
                        saveParamDerivs << "energyParamDerivs[get_global_id(0)*" << numDerivs << "+" << index << "] += energyParamDeriv" << i << ";\n";
            }
        }
Peter Eastman's avatar
Peter Eastman committed
4039
4040
4041
4042
4043
4044
4045
        map<string, string> variables;
        variables["x"] = "pos.x";
        variables["y"] = "pos.y";
        variables["z"] = "pos.z";
        for (int i = 0; i < force.getNumPerParticleParameters(); i++)
            variables[force.getPerParticleParameterName(i)] = "params"+params->getParameterSuffix(i, "[index]");
        for (int i = 0; i < force.getNumGlobalParameters(); i++)
4046
            variables[force.getGlobalParameterName(i)] = "globals["+cl.intToString(i)+"]";
Peter Eastman's avatar
Peter Eastman committed
4047
4048
        for (int i = 0; i < force.getNumComputedValues(); i++)
            variables[computedValueNames[i]] = "values"+computedValues->getParameterSuffix(i, "[index]");
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
        if (needParameterGradient) {
            for (int i = 1; i < force.getNumComputedValues(); i++) {
                string is = cl.intToString(i);
                compute << "real4 dV"<<is<<"dR = (real4) 0;\n";
                for (int j = 1; j < i; j++) {
                    if (!isZeroExpression(valueDerivExpressions[i][j])) {
                        map<string, Lepton::ParsedExpression> derivExpressions;
                        string js = cl.intToString(j);
                        derivExpressions["real dV"+is+"dV"+js+" = "] = valueDerivExpressions[i][j];
                        compute << cl.getExpressionUtilities().createExpressions(derivExpressions, variables, functionList, functionDefinitions, "temp_"+is+"_"+js);
                        compute << "dV"<<is<<"dR += dV"<<is<<"dV"<<js<<"*dV"<<js<<"dR;\n";
                    }
4061
                }
4062
4063
4064
4065
4066
4067
4068
4069
                map<string, Lepton::ParsedExpression> gradientExpressions;
                if (!isZeroExpression(valueGradientExpressions[i][0]))
                    gradientExpressions["dV"+is+"dR.x += "] = valueGradientExpressions[i][0];
                if (!isZeroExpression(valueGradientExpressions[i][1]))
                    gradientExpressions["dV"+is+"dR.y += "] = valueGradientExpressions[i][1];
                if (!isZeroExpression(valueGradientExpressions[i][2]))
                    gradientExpressions["dV"+is+"dR.z += "] = valueGradientExpressions[i][2];
                compute << cl.getExpressionUtilities().createExpressions(gradientExpressions, variables, functionList, functionDefinitions, "temp");
4070
            }
4071
4072
            for (int i = 1; i < force.getNumComputedValues(); i++)
                compute << "force -= deriv"<<energyDerivs->getParameterSuffix(i)<<"*dV"<<i<<"dR;\n";
Peter Eastman's avatar
Peter Eastman committed
4073
        }
4074
4075
4076
4077
        if (needEnergyParamDerivs)
            for (int i = 0; i < force.getNumComputedValues(); i++)
                for (int j = 0; j < dValuedParam.size(); j++)
                    compute << "energyParamDeriv"<<j<<" += deriv"<<energyDerivs->getParameterSuffix(i)<<"*dValuedParam_"<<i<<"_"<<j<<"[index];\n";
Peter Eastman's avatar
Peter Eastman committed
4078
4079
4080
        map<string, string> replacements;
        replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
        replacements["COMPUTE_FORCES"] = compute.str();
4081
4082
        replacements["INIT_PARAM_DERIVS"] = initParamDerivs.str();
        replacements["SAVE_PARAM_DERIVS"] = saveParamDerivs.str();
Peter Eastman's avatar
Peter Eastman committed
4083
        map<string, string> defines;
4084
        defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
Peter Eastman's avatar
Peter Eastman committed
4085
4086
4087
        cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customGBGradientChainRule, replacements), defines);
        gradientChainRuleKernel = cl::Kernel(program, "computeGradientChainRuleTerms");
    }
4088
    {
peastman's avatar
peastman committed
4089
        // Create the code to calculate chain rule terms as part of the default nonbonded kernel.
4090

4091
        vector<pair<ExpressionTreeNode, string> > globalVariables;
4092
4093
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
4094
            string value = "globals["+cl.intToString(i)+"]";
4095
            globalVariables.push_back(makeVariable(name, prefix+value));
4096
        }
4097
        vector<pair<ExpressionTreeNode, string> > variables = globalVariables;
4098
        map<string, string> rename;
4099
4100
4101
4102
        ExpressionTreeNode rnode(new Operation::Variable("r"));
        variables.push_back(make_pair(rnode, "r"));
        variables.push_back(make_pair(ExpressionTreeNode(new Operation::Square(), rnode), "r2"));
        variables.push_back(make_pair(ExpressionTreeNode(new Operation::Reciprocal(), rnode), "invR"));
4103
4104
        for (int i = 0; i < force.getNumPerParticleParameters(); i++) {
            const string& name = force.getPerParticleParameterName(i);
4105
4106
            variables.push_back(makeVariable(name+"1", prefix+"params"+params->getParameterSuffix(i, "1")));
            variables.push_back(makeVariable(name+"2", prefix+"params"+params->getParameterSuffix(i, "2")));
Peter Eastman's avatar
Peter Eastman committed
4107
4108
            rename[name+"1"] = name+"2";
            rename[name+"2"] = name+"1";
4109
4110
4111
4112
        }
        map<string, Lepton::ParsedExpression> derivExpressions;
        stringstream chainSource;
        Lepton::ParsedExpression dVdR = Lepton::Parser::parse(computedValueExpressions[0], functions).differentiate("r").optimize();
4113
4114
        derivExpressions["real dV0dR1 = "] = dVdR;
        derivExpressions["real dV0dR2 = "] = dVdR.renameVariables(rename);
4115
        chainSource << cl.getExpressionUtilities().createExpressions(derivExpressions, variables, functionList, functionDefinitions, prefix+"temp0_");
Peter Eastman's avatar
Peter Eastman committed
4116
4117
4118
4119
4120
4121
4122
        if (needChainForValue[0]) {
            if (useExclusionsForValue)
                chainSource << "if (!isExcluded) {\n";
            chainSource << "tempForce -= dV0dR1*" << prefix << "dEdV" << energyDerivs->getParameterSuffix(0, "1") << ";\n";
            chainSource << "tempForce -= dV0dR2*" << prefix << "dEdV" << energyDerivs->getParameterSuffix(0, "2") << ";\n";
            if (useExclusionsForValue)
                chainSource << "}\n";
4123
        }
Peter Eastman's avatar
Peter Eastman committed
4124
4125
4126
4127
        for (int i = 1; i < force.getNumComputedValues(); i++) {
            if (needChainForValue[i]) {
                chainSource << "tempForce -= dV0dR1*" << prefix << "dEdV" << energyDerivs->getParameterSuffix(i, "1") << ";\n";
                chainSource << "tempForce -= dV0dR2*" << prefix << "dEdV" << energyDerivs->getParameterSuffix(i, "2") << ";\n";
4128
            }
4129
4130
        }
        map<string, string> replacements;
Peter Eastman's avatar
Peter Eastman committed
4131
4132
        string chainStr = chainSource.str();
        replacements["COMPUTE_FORCE"] = chainStr;
4133
        string source = cl.replaceStrings(OpenCLKernelSources::customGBChainRule, replacements);
4134
4135
        vector<OpenCLNonbondedUtilities::ParameterInfo> parameters;
        vector<OpenCLNonbondedUtilities::ParameterInfo> arguments;
4136
4137
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
4138
            string paramName = prefix+"params"+cl.intToString(i+1);
Peter Eastman's avatar
Peter Eastman committed
4139
4140
            if (chainStr.find(paramName+"1") != chainStr.npos || chainStr.find(paramName+"2") != chainStr.npos)
                parameters.push_back(OpenCLNonbondedUtilities::ParameterInfo(paramName, buffer.getComponentType(), buffer.getNumComponents(), buffer.getSize(), buffer.getMemory()));
4141
4142
4143
        }
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
            const OpenCLNonbondedUtilities::ParameterInfo& buffer = computedValues->getBuffers()[i];
4144
            string paramName = prefix+"values"+cl.intToString(i+1);
Peter Eastman's avatar
Peter Eastman committed
4145
4146
            if (chainStr.find(paramName+"1") != chainStr.npos || chainStr.find(paramName+"2") != chainStr.npos)
                parameters.push_back(OpenCLNonbondedUtilities::ParameterInfo(paramName, buffer.getComponentType(), buffer.getNumComponents(), buffer.getSize(), buffer.getMemory()));
4147
        }
4148
        for (int i = 0; i < (int) energyDerivChain->getBuffers().size(); i++) {
Peter Eastman's avatar
Peter Eastman committed
4149
            if (needChainForValue[i]) { 
4150
                const OpenCLNonbondedUtilities::ParameterInfo& buffer = energyDerivChain->getBuffers()[i];
4151
                string paramName = prefix+"dEdV"+cl.intToString(i+1);
Peter Eastman's avatar
Peter Eastman committed
4152
4153
                parameters.push_back(OpenCLNonbondedUtilities::ParameterInfo(paramName, buffer.getComponentType(), buffer.getNumComponents(), buffer.getSize(), buffer.getMemory()));
            }
4154
        }
peastman's avatar
peastman committed
4155
4156
4157
        if (globals.isInitialized()) {
            globals.upload(globalParamValues);
            arguments.push_back(OpenCLNonbondedUtilities::ParameterInfo(prefix+"globals", "float", 1, sizeof(cl_float), globals.getDeviceBuffer()));
4158
        }
4159
        cl.getNonbondedUtilities().addInteraction(useCutoff, usePeriodic, force.getNumExclusions() > 0, cutoff, exclusionList, source, force.getForceGroup());
peastman's avatar
peastman committed
4160
4161
4162
4163
        for (auto param : parameters)
            cl.getNonbondedUtilities().addParameter(param);
        for (auto arg : arguments)
            cl.getNonbondedUtilities().addArgument(arg);
4164
    }
4165
4166
    info = new ForceInfo(cl.getNonbondedUtilities().getNumForceBuffers(), force);
    cl.addForce(info);
4167
    if (useLong)
peastman's avatar
peastman committed
4168
        cl.addAutoclearBuffer(longEnergyDerivs);
Peter Eastman's avatar
Peter Eastman committed
4169
    else {
peastman's avatar
peastman committed
4170
        for (auto& buffer : energyDerivs->getBuffers())
4171
            cl.addAutoclearBuffer(buffer.getMemory(), buffer.getSize()*energyDerivs->getNumObjects());
Peter Eastman's avatar
Peter Eastman committed
4172
    }
4173
4174
}

4175
double OpenCLCalcCustomGBForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
4176
    bool deviceIsCpu = (cl.getDevice().getInfo<CL_DEVICE_TYPE>() == CL_DEVICE_TYPE_CPU);
4177
    OpenCLNonbondedUtilities& nb = cl.getNonbondedUtilities();
4178
    int elementSize = (cl.getUseDoublePrecision() ? sizeof(cl_double) : sizeof(cl_float));
4179
4180
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
        
        // These two kernels can't be compiled in initialize(), because the nonbonded utilities object
        // has not yet been initialized then.

        {
            int numExclusionTiles = nb.getExclusionTiles().getSize();
            pairValueDefines["NUM_TILES_WITH_EXCLUSIONS"] = cl.intToString(numExclusionTiles);
            int numContexts = cl.getPlatformData().contexts.size();
            int startExclusionIndex = cl.getContextIndex()*numExclusionTiles/numContexts;
            int endExclusionIndex = (cl.getContextIndex()+1)*numExclusionTiles/numContexts;
            pairValueDefines["FIRST_EXCLUSION_TILE"] = cl.intToString(startExclusionIndex);
            pairValueDefines["LAST_EXCLUSION_TILE"] = cl.intToString(endExclusionIndex);
4193
            pairValueDefines["CUTOFF"] = cl.doubleToString(cutoff);
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
            cl::Program program = cl.createProgram(pairValueSrc, pairValueDefines);
            pairValueKernel = cl::Kernel(program, "computeN2Value");
            pairValueSrc = "";
            pairValueDefines.clear();
        }
        {
            int numExclusionTiles = nb.getExclusionTiles().getSize();
            pairEnergyDefines["NUM_TILES_WITH_EXCLUSIONS"] = cl.intToString(numExclusionTiles);
            int numContexts = cl.getPlatformData().contexts.size();
            int startExclusionIndex = cl.getContextIndex()*numExclusionTiles/numContexts;
            int endExclusionIndex = (cl.getContextIndex()+1)*numExclusionTiles/numContexts;
            pairEnergyDefines["FIRST_EXCLUSION_TILE"] = cl.intToString(startExclusionIndex);
            pairEnergyDefines["LAST_EXCLUSION_TILE"] = cl.intToString(endExclusionIndex);
4207
            pairEnergyDefines["CUTOFF"] = cl.doubleToString(cutoff);
4208
4209
4210
4211
4212
4213
4214
4215
            cl::Program program = cl.createProgram(pairEnergySrc, pairEnergyDefines);
            pairEnergyKernel = cl::Kernel(program, "computeN2Energy");
            pairEnergySrc = "";
            pairEnergyDefines.clear();
        }

        // Set arguments for kernels.
        
4216
        maxTiles = (nb.getUseCutoff() ? nb.getInteractingTiles().getSize() : 0);
4217
        bool useLong = cl.getSupports64BitGlobalAtomics();
4218
        if (useLong) {
peastman's avatar
peastman committed
4219
4220
4221
            longValueBuffers.initialize<cl_long>(cl, cl.getPaddedNumAtoms(), "customGBLongValueBuffers");
            cl.addAutoclearBuffer(longValueBuffers);
            cl.clearBuffer(longValueBuffers);
4222
4223
        }
        else {
peastman's avatar
peastman committed
4224
4225
4226
            valueBuffers.initialize(cl, cl.getPaddedNumAtoms()*nb.getNumForceBuffers(), elementSize, "customGBValueBuffers");
            cl.addAutoclearBuffer(valueBuffers);
            cl.clearBuffer(valueBuffers);
4227
        }
4228
4229
        int index = 0;
        pairValueKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
4230
        pairValueKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*4*elementSize, NULL);
4231
        pairValueKernel.setArg<cl::Buffer>(index++, cl.getNonbondedUtilities().getExclusions().getDeviceBuffer());
4232
        pairValueKernel.setArg<cl::Buffer>(index++, cl.getNonbondedUtilities().getExclusionTiles().getDeviceBuffer());
peastman's avatar
peastman committed
4233
        pairValueKernel.setArg<cl::Buffer>(index++, useLong ? longValueBuffers.getDeviceBuffer() : valueBuffers.getDeviceBuffer());
4234
        pairValueKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*elementSize, NULL);
4235
4236
4237
        if (nb.getUseCutoff()) {
            pairValueKernel.setArg<cl::Buffer>(index++, nb.getInteractingTiles().getDeviceBuffer());
            pairValueKernel.setArg<cl::Buffer>(index++, nb.getInteractionCount().getDeviceBuffer());
4238
            index += 5; // Periodic box size arguments are set when the kernel is executed.
4239
            pairValueKernel.setArg<cl_uint>(index++, maxTiles);
4240
            pairValueKernel.setArg<cl::Buffer>(index++, nb.getBlockCenters().getDeviceBuffer());
4241
            pairValueKernel.setArg<cl::Buffer>(index++, nb.getBlockBoundingBoxes().getDeviceBuffer());
4242
            pairValueKernel.setArg<cl::Buffer>(index++, nb.getInteractingAtoms().getDeviceBuffer());
4243
        }
4244
4245
        else
            pairValueKernel.setArg<cl_uint>(index++, cl.getNumAtomBlocks()*(cl.getNumAtomBlocks()+1)/2);
peastman's avatar
peastman committed
4246
4247
        if (globals.isInitialized())
            pairValueKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
4248
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
Peter Eastman's avatar
Peter Eastman committed
4249
4250
4251
4252
4253
            if (pairValueUsesParam[i]) {
                const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
                pairValueKernel.setArg<cl::Memory>(index++, buffer.getMemory());
                pairValueKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*buffer.getSize(), NULL);
            }
4254
        }
peastman's avatar
peastman committed
4255
4256
4257
        for (auto& d : dValue0dParam) {
            pairValueKernel.setArg<cl::Buffer>(index++, d.getDeviceBuffer());
            pairValueKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*d.getElementSize(), NULL);
4258
        }
peastman's avatar
peastman committed
4259
4260
        for (auto& function : tabulatedFunctions)
            pairValueKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
4261
        index = 0;
4262
4263
        perParticleValueKernel.setArg<cl_int>(index++, cl.getPaddedNumAtoms());
        perParticleValueKernel.setArg<cl_int>(index++, nb.getNumForceBuffers());
4264
        perParticleValueKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
4265
4266
4267
        perParticleValueKernel.setArg<cl::Buffer>(index++, useLong ? longValueBuffers.getDeviceBuffer() : valueBuffers.getDeviceBuffer());
        if (globals.isInitialized())
            perParticleValueKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
4268
4269
4270
4271
        for (auto& buffer : params->getBuffers())
            perParticleValueKernel.setArg<cl::Memory>(index++, buffer.getMemory());
        for (auto& buffer : computedValues->getBuffers())
            perParticleValueKernel.setArg<cl::Memory>(index++, buffer.getMemory());
4272
        for (int i = 0; i < dValuedParam.size(); i++) {
peastman's avatar
peastman committed
4273
            perParticleValueKernel.setArg<cl::Memory>(index++, dValue0dParam[i].getDeviceBuffer());
4274
4275
4276
            for (int j = 0; j < dValuedParam[i]->getBuffers().size(); j++)
                perParticleValueKernel.setArg<cl::Memory>(index++, dValuedParam[i]->getBuffers()[j].getMemory());
        }
peastman's avatar
peastman committed
4277
4278
        for (auto& function : tabulatedFunctions)
            perParticleValueKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
4279
        index = 0;
4280
        pairEnergyKernel.setArg<cl::Buffer>(index++, useLong ? cl.getLongForceBuffer().getDeviceBuffer() : cl.getForceBuffers().getDeviceBuffer());
4281
        pairEnergyKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
4282
        pairEnergyKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*4*elementSize, NULL);
4283
        pairEnergyKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
4284
        pairEnergyKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*4*elementSize, NULL);
4285
        pairEnergyKernel.setArg<cl::Buffer>(index++, cl.getNonbondedUtilities().getExclusions().getDeviceBuffer());
4286
        pairEnergyKernel.setArg<cl::Buffer>(index++, cl.getNonbondedUtilities().getExclusionTiles().getDeviceBuffer());
4287
        index++; // Whether to include energy.
4288
4289
4290
        if (nb.getUseCutoff()) {
            pairEnergyKernel.setArg<cl::Buffer>(index++, nb.getInteractingTiles().getDeviceBuffer());
            pairEnergyKernel.setArg<cl::Buffer>(index++, nb.getInteractionCount().getDeviceBuffer());
4291
            index += 5; // Periodic box size arguments are set when the kernel is executed.
4292
            pairEnergyKernel.setArg<cl_uint>(index++, maxTiles);
4293
            pairEnergyKernel.setArg<cl::Buffer>(index++, nb.getBlockCenters().getDeviceBuffer());
4294
            pairEnergyKernel.setArg<cl::Buffer>(index++, nb.getBlockBoundingBoxes().getDeviceBuffer());
4295
            pairEnergyKernel.setArg<cl::Buffer>(index++, nb.getInteractingAtoms().getDeviceBuffer());
4296
        }
4297
4298
        else
            pairEnergyKernel.setArg<cl_uint>(index++, cl.getNumAtomBlocks()*(cl.getNumAtomBlocks()+1)/2);
peastman's avatar
peastman committed
4299
4300
        if (globals.isInitialized())
            pairEnergyKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
4301
        for (int i = 0; i < (int) params->getBuffers().size(); i++) {
Peter Eastman's avatar
Peter Eastman committed
4302
4303
4304
4305
4306
            if (pairEnergyUsesParam[i]) {
                const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
                pairEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
                pairEnergyKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*buffer.getSize(), NULL);
            }
4307
4308
        }
        for (int i = 0; i < (int) computedValues->getBuffers().size(); i++) {
Peter Eastman's avatar
Peter Eastman committed
4309
4310
4311
4312
4313
            if (pairEnergyUsesValue[i]) {
                const OpenCLNonbondedUtilities::ParameterInfo& buffer = computedValues->getBuffers()[i];
                pairEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
                pairEnergyKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*buffer.getSize(), NULL);
            }
4314
        }
4315
        if (useLong) {
peastman's avatar
peastman committed
4316
            pairEnergyKernel.setArg<cl::Memory>(index++, longEnergyDerivs.getDeviceBuffer());
4317
            for (int i = 0; i < numComputedValues; ++i)
4318
                pairEnergyKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*elementSize, NULL);
4319
4320
        }
        else {
peastman's avatar
peastman committed
4321
            for (auto& buffer : energyDerivs->getBuffers()) {
4322
4323
4324
                pairEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
                pairEnergyKernel.setArg(index++, (deviceIsCpu ? OpenCLContext::TileSize : nb.getForceThreadBlockSize())*buffer.getSize(), NULL);
            }
4325
        }
4326
4327
        if (needEnergyParamDerivs)
            pairEnergyKernel.setArg<cl::Memory>(index++, cl.getEnergyParamDerivBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
4328
4329
        for (auto& function : tabulatedFunctions)
            pairEnergyKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
4330
4331
4332
        index = 0;
        perParticleEnergyKernel.setArg<cl_int>(index++, cl.getPaddedNumAtoms());
        perParticleEnergyKernel.setArg<cl_int>(index++, nb.getNumForceBuffers());
4333
        perParticleEnergyKernel.setArg<cl::Buffer>(index++, cl.getForceBuffers().getDeviceBuffer());
4334
        perParticleEnergyKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
4335
        perParticleEnergyKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
4336
4337
        if (globals.isInitialized())
            perParticleEnergyKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
4338
4339
4340
4341
4342
4343
4344
4345
        for (auto& buffer : params->getBuffers())
            perParticleEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
        for (auto& buffer : computedValues->getBuffers())
            perParticleEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
        for (auto& buffer : energyDerivs->getBuffers())
            perParticleEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
        for (auto& buffer : energyDerivChain->getBuffers())
            perParticleEnergyKernel.setArg<cl::Memory>(index++, buffer.getMemory());
4346
        if (useLong)
peastman's avatar
peastman committed
4347
            perParticleEnergyKernel.setArg<cl::Memory>(index++, longEnergyDerivs.getDeviceBuffer());
4348
4349
        if (needEnergyParamDerivs)
            perParticleEnergyKernel.setArg<cl::Memory>(index++, cl.getEnergyParamDerivBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
4350
4351
        for (auto& function : tabulatedFunctions)
            perParticleEnergyKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
4352
        if (needParameterGradient || needEnergyParamDerivs) {
Peter Eastman's avatar
Peter Eastman committed
4353
4354
4355
            index = 0;
            gradientChainRuleKernel.setArg<cl::Buffer>(index++, cl.getForceBuffers().getDeviceBuffer());
            gradientChainRuleKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
4356
4357
            if (globals.isInitialized())
                gradientChainRuleKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
4358
4359
4360
4361
4362
4363
            for (auto& buffer : params->getBuffers())
                gradientChainRuleKernel.setArg<cl::Memory>(index++, buffer.getMemory());
            for (auto& buffer : computedValues->getBuffers())
                gradientChainRuleKernel.setArg<cl::Memory>(index++, buffer.getMemory());
            for (auto& buffer : energyDerivs->getBuffers())
                gradientChainRuleKernel.setArg<cl::Memory>(index++, buffer.getMemory());
4364
4365
            if (needEnergyParamDerivs) {
                gradientChainRuleKernel.setArg<cl::Buffer>(index++, cl.getEnergyParamDerivBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
4366
4367
4368
                for (auto d : dValuedParam)
                    for (auto& buffer : d->getBuffers())
                        gradientChainRuleKernel.setArg<cl::Memory>(index++, buffer.getMemory());
4369
            }
Peter Eastman's avatar
Peter Eastman committed
4370
        }
4371
    }
peastman's avatar
peastman committed
4372
    if (globals.isInitialized()) {
4373
        bool changed = false;
4374
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
4375
4376
4377
4378
4379
4380
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
4381
            globals.upload(globalParamValues);
4382
    }
4383
    pairEnergyKernel.setArg<cl_int>(7, includeEnergy);
4384
    if (nb.getUseCutoff()) {
4385
        setPeriodicBoxArgs(cl, pairValueKernel, 8);
4386
        setPeriodicBoxArgs(cl, pairEnergyKernel, 10);
4387
4388
        if (maxTiles < nb.getInteractingTiles().getSize()) {
            maxTiles = nb.getInteractingTiles().getSize();
4389
            pairValueKernel.setArg<cl::Buffer>(6, nb.getInteractingTiles().getDeviceBuffer());
4390
4391
            pairValueKernel.setArg<cl_uint>(13, maxTiles);
            pairValueKernel.setArg<cl::Buffer>(16, nb.getInteractingAtoms().getDeviceBuffer());
4392
4393
4394
            pairEnergyKernel.setArg<cl::Buffer>(8, nb.getInteractingTiles().getDeviceBuffer());
            pairEnergyKernel.setArg<cl_uint>(15, maxTiles);
            pairEnergyKernel.setArg<cl::Buffer>(18, nb.getInteractingAtoms().getDeviceBuffer());
4395
        }
4396
    }
4397
    cl.executeKernel(pairValueKernel, nb.getNumForceThreadBlocks()*nb.getForceThreadBlockSize(), nb.getForceThreadBlockSize());
4398
    cl.executeKernel(perParticleValueKernel, cl.getPaddedNumAtoms());
4399
    cl.executeKernel(pairEnergyKernel, nb.getNumForceThreadBlocks()*nb.getForceThreadBlockSize(), nb.getForceThreadBlockSize());
4400
    cl.executeKernel(perParticleEnergyKernel, cl.getPaddedNumAtoms());
4401
    if (needParameterGradient || needEnergyParamDerivs)
Peter Eastman's avatar
Peter Eastman committed
4402
        cl.executeKernel(gradientChainRuleKernel, cl.getPaddedNumAtoms());
4403
4404
4405
    return 0.0;
}

4406
4407
4408
4409
4410
4411
4412
void OpenCLCalcCustomGBForceKernel::copyParametersToContext(ContextImpl& context, const CustomGBForce& force) {
    int numParticles = force.getNumParticles();
    if (numParticles != cl.getNumAtoms())
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
    
    // Record the per-particle parameters.
    
4413
    vector<vector<cl_float> > paramVector(cl.getPaddedNumAtoms(), vector<cl_float>(force.getNumPerParticleParameters(), 0));
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
    vector<double> parameters;
    for (int i = 0; i < numParticles; i++) {
        force.getParticleParameters(i, parameters);
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
4424
    cl.invalidateMolecules(info);
4425
4426
}

4427
class OpenCLCalcCustomExternalForceKernel::ForceInfo : public OpenCLForceInfo {
4428
public:
4429
    ForceInfo(const CustomExternalForce& force, int numParticles) : OpenCLForceInfo(0), force(force), indices(numParticles, -1) {
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
        vector<double> params;
        for (int i = 0; i < force.getNumParticles(); i++) {
            int particle;
            force.getParticleParameters(i, particle, params);
            indices[particle] = i;
        }
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        particle1 = indices[particle1];
        particle2 = indices[particle2];
        if (particle1 == -1 && particle2 == -1)
            return true;
        if (particle1 == -1 || particle2 == -1)
            return false;
        int temp;
        vector<double> params1;
        vector<double> params2;
        force.getParticleParameters(particle1, temp, params1);
        force.getParticleParameters(particle2, temp, params2);
4449
        for (int i = 0; i < (int) params1.size(); i++)
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
            if (params1[i] != params2[i])
                return false;
        return true;
    }
private:
    const CustomExternalForce& force;
    vector<int> indices;
};

OpenCLCalcCustomExternalForceKernel::~OpenCLCalcCustomExternalForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomExternalForceKernel::initialize(const System& system, const CustomExternalForce& force) {
4465
4466
4467
4468
4469
4470
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumParticles()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumParticles()/numContexts;
    numParticles = endIndex-startIndex;
    if (numParticles == 0)
        return;
4471
    vector<vector<int> > atoms(numParticles, vector<int>(1));
4472
4473
    params = new OpenCLParameterSet(cl, force.getNumPerParticleParameters(), numParticles, "customExternalParams");
    vector<vector<cl_float> > paramVector(numParticles);
4474
4475
    for (int i = 0; i < numParticles; i++) {
        vector<double> parameters;
4476
        force.getParticleParameters(startIndex+i, atoms[i][0], parameters);
4477
        paramVector[i].resize(parameters.size());
4478
        for (int j = 0; j < (int) parameters.size(); j++)
4479
            paramVector[i][j] = (cl_float) parameters[j];
4480
    }
4481
    params->setParameterValues(paramVector);
4482
4483
    info = new ForceInfo(force, system.getNumParticles());
    cl.addForce(info);
4484
4485
4486
4487
4488
4489
4490
4491
4492

    // Record information for the expressions.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
4493
4494
4495
    map<string, Lepton::CustomFunction*> customFunctions;
    customFunctions["periodicdistance"] = cl.getExpressionUtilities().getPeriodicDistancePlaceholder();
    Lepton::ParsedExpression energyExpression = Lepton::Parser::parse(force.getEnergyFunction(), customFunctions).optimize();
4496
4497
4498
4499
4500
    Lepton::ParsedExpression forceExpressionX = energyExpression.differentiate("x").optimize();
    Lepton::ParsedExpression forceExpressionY = energyExpression.differentiate("y").optimize();
    Lepton::ParsedExpression forceExpressionZ = energyExpression.differentiate("z").optimize();
    map<string, Lepton::ParsedExpression> expressions;
    expressions["energy += "] = energyExpression;
4501
4502
4503
    expressions["real dEdX = "] = forceExpressionX;
    expressions["real dEdY = "] = forceExpressionY;
    expressions["real dEdZ = "] = forceExpressionZ;
4504
4505
4506
4507

    // Create the kernels.

    map<string, string> variables;
4508
4509
4510
    variables["x"] = "pos1.x";
    variables["y"] = "pos1.y";
    variables["z"] = "pos1.z";
4511
4512
    for (int i = 0; i < force.getNumPerParticleParameters(); i++) {
        const string& name = force.getPerParticleParameterName(i);
4513
        variables[name] = "particleParams"+params->getParameterSuffix(i);
4514
    }
4515
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
4516
4517
4518
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customExternalGlobals", CL_MEM_READ_ONLY);
        globals.upload(globalParamValues);
        string argName = cl.getBondedUtilities().addArgument(globals.getDeviceBuffer(), "float");
4519
4520
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
4521
            string value = argName+"["+cl.intToString(i)+"]";
4522
4523
            variables[name] = value;
        }
4524
4525
    }
    stringstream compute;
4526
4527
    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
4528
4529
        string argName = cl.getBondedUtilities().addArgument(buffer.getMemory(), buffer.getType());
        compute<<buffer.getType()<<" particleParams"<<(i+1)<<" = "<<argName<<"[index];\n";
4530
    }
peastman's avatar
peastman committed
4531
4532
    vector<const TabulatedFunction*> functions;
    vector<pair<string, string> > functionNames;
4533
    compute << cl.getExpressionUtilities().createExpressions(expressions, variables, functions, functionNames, "temp");
4534
    map<string, string> replacements;
4535
    replacements["COMPUTE_FORCE"] = compute.str();
4536
    cl.getBondedUtilities().addInteraction(atoms, cl.replaceStrings(OpenCLKernelSources::customExternalForce, replacements), force.getForceGroup());
4537
4538
}

4539
double OpenCLCalcCustomExternalForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
peastman's avatar
peastman committed
4540
    if (globals.isInitialized()) {
4541
        bool changed = false;
4542
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
4543
4544
4545
4546
4547
4548
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
4549
            globals.upload(globalParamValues);
4550
4551
    }
    return 0.0;
4552
}
4553

4554
4555
4556
4557
4558
4559
void OpenCLCalcCustomExternalForceKernel::copyParametersToContext(ContextImpl& context, const CustomExternalForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumParticles()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumParticles()/numContexts;
    if (numParticles != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
4560
4561
    if (numParticles == 0)
        return;
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
    
    // Record the per-particle parameters.
    
    vector<vector<cl_float> > paramVector(numParticles);
    vector<double> parameters;
    for (int i = 0; i < numParticles; i++) {
        int particle;
        force.getParticleParameters(startIndex+i, particle, parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
4578
    cl.invalidateMolecules(info);
4579
4580
}

4581
class OpenCLCalcCustomHbondForceKernel::ForceInfo : public OpenCLForceInfo {
4582
public:
4583
    ForceInfo(int requiredBuffers, const CustomHbondForce& force) : OpenCLForceInfo(requiredBuffers), force(force) {
4584
4585
4586
4587
4588
4589
4590
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        return true;
    }
    int getNumParticleGroups() {
        return force.getNumDonors()+force.getNumAcceptors()+force.getNumExclusions();
    }
Peter Eastman's avatar
Peter Eastman committed
4591
    void getParticlesInGroup(int index, vector<int>& particles) {
4592
4593
4594
4595
        int p1, p2, p3;
        vector<double> parameters;
        if (index < force.getNumDonors()) {
            force.getDonorParameters(index, p1, p2, p3, parameters);
4596
4597
4598
4599
4600
4601
            particles.clear();
            particles.push_back(p1);
            if (p2 > -1)
                particles.push_back(p2);
            if (p3 > -1)
                particles.push_back(p3);
4602
4603
4604
4605
4606
            return;
        }
        index -= force.getNumDonors();
        if (index < force.getNumAcceptors()) {
            force.getAcceptorParameters(index, p1, p2, p3, parameters);
4607
4608
4609
4610
4611
4612
            particles.clear();
            particles.push_back(p1);
            if (p2 > -1)
                particles.push_back(p2);
            if (p3 > -1)
                particles.push_back(p3);
4613
4614
4615
4616
4617
            return;
        }
        index -= force.getNumAcceptors();
        int donor, acceptor;
        force.getExclusionParticles(index, donor, acceptor);
4618
        particles.clear();
4619
        force.getDonorParameters(donor, p1, p2, p3, parameters);
4620
4621
4622
4623
4624
        particles.push_back(p1);
        if (p2 > -1)
            particles.push_back(p2);
        if (p3 > -1)
            particles.push_back(p3);
4625
        force.getAcceptorParameters(acceptor, p1, p2, p3, parameters);
4626
4627
4628
4629
4630
        particles.push_back(p1);
        if (p2 > -1)
            particles.push_back(p2);
        if (p3 > -1)
            particles.push_back(p3);
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
    }
    bool areGroupsIdentical(int group1, int group2) {
        int p1, p2, p3;
        vector<double> params1, params2;
        if (group1 < force.getNumDonors() && group2 < force.getNumDonors()) {
            force.getDonorParameters(group1, p1, p2, p3, params1);
            force.getDonorParameters(group2, p1, p2, p3, params2);
            return (params1 == params2 && params1 == params2);
        }
        if (group1 < force.getNumDonors() || group2 < force.getNumDonors())
            return false;
        group1 -= force.getNumDonors();
        group2 -= force.getNumDonors();
        if (group1 < force.getNumAcceptors() && group2 < force.getNumAcceptors()) {
            force.getAcceptorParameters(group1, p1, p2, p3, params1);
            force.getAcceptorParameters(group2, p1, p2, p3, params2);
            return (params1 == params2 && params1 == params2);
        }
        if (group1 < force.getNumAcceptors() || group2 < force.getNumAcceptors())
            return false;
        return true;
    }
private:
    const CustomHbondForce& force;
};

OpenCLCalcCustomHbondForceKernel::~OpenCLCalcCustomHbondForceKernel() {
    if (donorParams != NULL)
        delete donorParams;
    if (acceptorParams != NULL)
        delete acceptorParams;
}

4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
static void addDonorAndAcceptorCode(stringstream& computeDonor, stringstream& computeAcceptor, const string& value) {
    computeDonor << value;
    computeAcceptor << value;
}

static void applyDonorAndAcceptorForces(stringstream& applyToDonor, stringstream& applyToAcceptor, int atom, const string& value) {
    string forceNames[] = {"f1", "f2", "f3"};
    if (atom < 3)
        applyToAcceptor << forceNames[atom]<<".xyz += "<<value<<";\n";
    else
        applyToDonor << forceNames[atom-3]<<".xyz += "<<value<<";\n";
}
4676

4677
void OpenCLCalcCustomHbondForceKernel::initialize(const System& system, const CustomHbondForce& force) {
4678
4679
    // Record the lists of donors and acceptors, and the parameters for each one.

4680
4681
4682
4683
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumDonors()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumDonors()/numContexts;
    numDonors = endIndex-startIndex;
4684
    numAcceptors = force.getNumAcceptors();
4685
4686
    if (numDonors == 0 || numAcceptors == 0)
        return;
4687
    int numParticles = system.getNumParticles();
peastman's avatar
peastman committed
4688
4689
    donors.initialize<mm_int4>(cl, numDonors, "customHbondDonors");
    acceptors.initialize<mm_int4>(cl, numAcceptors, "customHbondAcceptors");
4690
4691
4692
    donorParams = new OpenCLParameterSet(cl, force.getNumPerDonorParameters(), numDonors, "customHbondDonorParameters");
    acceptorParams = new OpenCLParameterSet(cl, force.getNumPerAcceptorParameters(), numAcceptors, "customHbondAcceptorParameters");
    if (force.getNumGlobalParameters() > 0)
peastman's avatar
peastman committed
4693
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customHbondGlobals", CL_MEM_READ_ONLY);
4694
4695
4696
4697
    vector<vector<cl_float> > donorParamVector(numDonors);
    vector<mm_int4> donorVector(numDonors);
    for (int i = 0; i < numDonors; i++) {
        vector<double> parameters;
4698
        force.getDonorParameters(startIndex+i, donorVector[i].x, donorVector[i].y, donorVector[i].z, parameters);
4699
4700
4701
4702
        donorParamVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            donorParamVector[i][j] = (cl_float) parameters[j];
    }
peastman's avatar
peastman committed
4703
    donors.upload(donorVector);
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
    donorParams->setParameterValues(donorParamVector);
    vector<vector<cl_float> > acceptorParamVector(numAcceptors);
    vector<mm_int4> acceptorVector(numAcceptors);
    for (int i = 0; i < numAcceptors; i++) {
        vector<double> parameters;
        force.getAcceptorParameters(i, acceptorVector[i].x, acceptorVector[i].y, acceptorVector[i].z, parameters);
        acceptorParamVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            acceptorParamVector[i][j] = (cl_float) parameters[j];
    }
peastman's avatar
peastman committed
4714
    acceptors.upload(acceptorVector);
4715
4716
    acceptorParams->setParameterValues(acceptorParamVector);

4717
    // Select an output buffer index for each donor and acceptor.
4718

peastman's avatar
peastman committed
4719
4720
    donorBufferIndices.initialize<mm_int4>(cl, numDonors, "customHbondDonorBuffers");
    acceptorBufferIndices.initialize<mm_int4>(cl, numAcceptors, "customHbondAcceptorBuffers");
4721
4722
    vector<mm_int4> donorBufferVector(numDonors);
    vector<mm_int4> acceptorBufferVector(numAcceptors);
4723
    vector<int> donorBufferCounter(numParticles, 0);
4724
    for (int i = 0; i < numDonors; i++)
4725
4726
4727
        donorBufferVector[i] = mm_int4(donorVector[i].x > -1 ? donorBufferCounter[donorVector[i].x]++ : 0,
                                       donorVector[i].y > -1 ? donorBufferCounter[donorVector[i].y]++ : 0,
                                       donorVector[i].z > -1 ? donorBufferCounter[donorVector[i].z]++ : 0, 0);
4728
    vector<int> acceptorBufferCounter(numParticles, 0);
4729
    for (int i = 0; i < numAcceptors; i++)
4730
4731
4732
        acceptorBufferVector[i] = mm_int4(acceptorVector[i].x > -1 ? acceptorBufferCounter[acceptorVector[i].x]++ : 0,
                                       acceptorVector[i].y > -1 ? acceptorBufferCounter[acceptorVector[i].y]++ : 0,
                                       acceptorVector[i].z > -1 ? acceptorBufferCounter[acceptorVector[i].z]++ : 0, 0);
peastman's avatar
peastman committed
4733
4734
    donorBufferIndices.upload(donorBufferVector);
    acceptorBufferIndices.upload(acceptorBufferVector);
4735
    int maxBuffers = 1;
peastman's avatar
peastman committed
4736
4737
4738
4739
    for (int i : donorBufferCounter)
        maxBuffers = max(maxBuffers, i);
    for (int i : acceptorBufferCounter)
        maxBuffers = max(maxBuffers, i);
4740
4741
    info = new ForceInfo(maxBuffers, force);
    cl.addForce(info);
4742
4743
4744

    // Record exclusions.

4745
4746
    vector<mm_int4> donorExclusionVector(numDonors, mm_int4(-1, -1, -1, -1));
    vector<mm_int4> acceptorExclusionVector(numAcceptors, mm_int4(-1, -1, -1, -1));
4747
4748
4749
    for (int i = 0; i < force.getNumExclusions(); i++) {
        int donor, acceptor;
        force.getExclusionParticles(i, donor, acceptor);
4750
4751
4752
        if (donor < startIndex || donor >= endIndex)
            continue;
        donor -= startIndex;
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
        if (donorExclusionVector[donor].x == -1)
            donorExclusionVector[donor].x = acceptor;
        else if (donorExclusionVector[donor].y == -1)
            donorExclusionVector[donor].y = acceptor;
        else if (donorExclusionVector[donor].z == -1)
            donorExclusionVector[donor].z = acceptor;
        else if (donorExclusionVector[donor].w == -1)
            donorExclusionVector[donor].w = acceptor;
        else
            throw OpenMMException("CustomHbondForce: OpenCLPlatform does not support more than four exclusions per donor");
        if (acceptorExclusionVector[acceptor].x == -1)
            acceptorExclusionVector[acceptor].x = donor;
        else if (acceptorExclusionVector[acceptor].y == -1)
            acceptorExclusionVector[acceptor].y = donor;
        else if (acceptorExclusionVector[acceptor].z == -1)
            acceptorExclusionVector[acceptor].z = donor;
        else if (acceptorExclusionVector[acceptor].w == -1)
            acceptorExclusionVector[acceptor].w = donor;
        else
            throw OpenMMException("CustomHbondForce: OpenCLPlatform does not support more than four exclusions per acceptor");
4773
    }
peastman's avatar
peastman committed
4774
4775
4776
4777
    donorExclusions.initialize<mm_int4>(cl, numDonors, "customHbondDonorExclusions");
    acceptorExclusions.initialize<mm_int4>(cl, numAcceptors, "customHbondAcceptorExclusions");
    donorExclusions.upload(donorExclusionVector);
    acceptorExclusions.upload(acceptorExclusionVector);
4778
4779
4780
4781
4782

    // Record the tabulated functions.

    map<string, Lepton::CustomFunction*> functions;
    vector<pair<string, string> > functionDefinitions;
4783
    vector<const TabulatedFunction*> functionList;
4784
    stringstream tableArgs;
peastman's avatar
peastman committed
4785
    tabulatedFunctions.resize(force.getNumFunctions());
4786
    for (int i = 0; i < force.getNumFunctions(); i++) {
4787
4788
        functionList.push_back(&force.getTabulatedFunction(i));
        string name = force.getTabulatedFunctionName(i);
4789
        string arrayName = "table"+cl.intToString(i);
4790
        functionDefinitions.push_back(make_pair(name, arrayName));
4791
        functions[name] = cl.getExpressionUtilities().getFunctionPlaceholder(force.getTabulatedFunction(i));
peastman's avatar
peastman committed
4792
        int width;
4793
        vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(force.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
4794
4795
        tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
        tabulatedFunctions[i].upload(f);
peastman's avatar
peastman committed
4796
4797
4798
4799
        tableArgs << ", __global const float";
        if (width > 1)
            tableArgs << width;
        tableArgs << "* restrict " << arrayName;
4800
4801
    }

4802
    // Record information about parameters.
4803
4804
4805
4806
4807
4808
4809

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
peastman's avatar
peastman committed
4810
4811
    if (globals.isInitialized())
        globals.upload(globalParamValues);
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
    map<string, string> variables;
    for (int i = 0; i < force.getNumPerDonorParameters(); i++) {
        const string& name = force.getPerDonorParameterName(i);
        variables[name] = "donorParams"+donorParams->getParameterSuffix(i);
    }
    for (int i = 0; i < force.getNumPerAcceptorParameters(); i++) {
        const string& name = force.getPerAcceptorParameterName(i);
        variables[name] = "acceptorParams"+acceptorParams->getParameterSuffix(i);
    }
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        const string& name = force.getGlobalParameterName(i);
4823
        variables[name] = "globals["+cl.intToString(i)+"]";
4824
    }
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839

    // Now to generate the kernel.  First, it needs to calculate all distances, angles,
    // and dihedrals the expression depends on.

    map<string, vector<int> > distances;
    map<string, vector<int> > angles;
    map<string, vector<int> > dihedrals;
    Lepton::ParsedExpression energyExpression = CustomHbondForceImpl::prepareExpression(force, functions, distances, angles, dihedrals);
    map<string, Lepton::ParsedExpression> forceExpressions;
    set<string> computedDeltas;
    computedDeltas.insert("D1A1");
    string atomNames[] = {"A1", "A2", "A3", "D1", "D2", "D3"};
    string atomNamesLower[] = {"a1", "a2", "a3", "d1", "d2", "d3"};
    stringstream computeDonor, computeAcceptor, extraArgs;
    int index = 0;
peastman's avatar
peastman committed
4840
4841
    for (auto& distance : distances) {
        const vector<int>& atoms = distance.second;
4842
4843
        string deltaName = atomNames[atoms[0]]+atomNames[atoms[1]];
        if (computedDeltas.count(deltaName) == 0) {
Peter Eastman's avatar
Peter Eastman committed
4844
            addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 delta"+deltaName+" = delta("+atomNamesLower[atoms[0]]+", "+atomNamesLower[atoms[1]]+", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n");
4845
4846
            computedDeltas.insert(deltaName);
        }
4847
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real r_"+deltaName+" = SQRT(delta"+deltaName+".w);\n");
peastman's avatar
peastman committed
4848
4849
4850
        variables[distance.first] = "r_"+deltaName;
        forceExpressions["real dEdDistance"+cl.intToString(index)+" = "] = energyExpression.differentiate(distance.first).optimize();
        index++;
4851
4852
    }
    index = 0;
peastman's avatar
peastman committed
4853
4854
    for (auto& angle : angles) {
        const vector<int>& atoms = angle.second;
4855
4856
4857
4858
        string deltaName1 = atomNames[atoms[1]]+atomNames[atoms[0]];
        string deltaName2 = atomNames[atoms[1]]+atomNames[atoms[2]];
        string angleName = "angle_"+atomNames[atoms[0]]+atomNames[atoms[1]]+atomNames[atoms[2]];
        if (computedDeltas.count(deltaName1) == 0) {
Peter Eastman's avatar
Peter Eastman committed
4859
            addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 delta"+deltaName1+" = delta("+atomNamesLower[atoms[1]]+", "+atomNamesLower[atoms[0]]+", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n");
4860
4861
4862
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
Peter Eastman's avatar
Peter Eastman committed
4863
            addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 delta"+deltaName2+" = delta("+atomNamesLower[atoms[1]]+", "+atomNamesLower[atoms[2]]+", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n");
4864
4865
            computedDeltas.insert(deltaName2);
        }
4866
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real "+angleName+" = computeAngle(delta"+deltaName1+", delta"+deltaName2+");\n");
peastman's avatar
peastman committed
4867
4868
4869
        variables[angle.first] = angleName;
        forceExpressions["real dEdAngle"+cl.intToString(index)+" = "] = energyExpression.differentiate(angle.first).optimize();
        index++;
4870
4871
    }
    index = 0;
peastman's avatar
peastman committed
4872
4873
    for (auto& dihedral : dihedrals) {
        const vector<int>& atoms = dihedral.second;
4874
4875
4876
4877
4878
4879
4880
        string deltaName1 = atomNames[atoms[0]]+atomNames[atoms[1]];
        string deltaName2 = atomNames[atoms[2]]+atomNames[atoms[1]];
        string deltaName3 = atomNames[atoms[2]]+atomNames[atoms[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        string dihedralName = "dihedral_"+atomNames[atoms[0]]+atomNames[atoms[1]]+atomNames[atoms[2]]+atomNames[atoms[3]];
        if (computedDeltas.count(deltaName1) == 0) {
Peter Eastman's avatar
Peter Eastman committed
4881
            addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 delta"+deltaName1+" = delta("+atomNamesLower[atoms[0]]+", "+atomNamesLower[atoms[1]]+", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n");
4882
4883
4884
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
Peter Eastman's avatar
Peter Eastman committed
4885
            addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 delta"+deltaName2+" = delta("+atomNamesLower[atoms[2]]+", "+atomNamesLower[atoms[1]]+", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n");
4886
4887
4888
            computedDeltas.insert(deltaName2);
        }
        if (computedDeltas.count(deltaName3) == 0) {
Peter Eastman's avatar
Peter Eastman committed
4889
            addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 delta"+deltaName3+" = delta("+atomNamesLower[atoms[2]]+", "+atomNamesLower[atoms[3]]+", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n");
4890
4891
            computedDeltas.insert(deltaName3);
        }
4892
4893
4894
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 "+crossName1+" = computeCross(delta"+deltaName1+", delta"+deltaName2+");\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 "+crossName2+" = computeCross(delta"+deltaName2+", delta"+deltaName3+");\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real "+dihedralName+" = computeAngle("+crossName1+", "+crossName2+");\n");
4895
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, dihedralName+" *= (delta"+deltaName1+".x*"+crossName2+".x + delta"+deltaName1+".y*"+crossName2+".y + delta"+deltaName1+".z*"+crossName2+".z < 0 ? -1 : 1);\n");
peastman's avatar
peastman committed
4896
4897
4898
        variables[dihedral.first] = dihedralName;
        forceExpressions["real dEdDihedral"+cl.intToString(index)+" = "] = energyExpression.differentiate(dihedral.first).optimize();
        index++;
4899
4900
4901
4902
    }

    // Next it needs to load parameters from global memory.

4903
    if (force.getNumGlobalParameters() > 0)
4904
        extraArgs << ", __global const float* restrict globals";
4905
4906
    for (int i = 0; i < (int) donorParams->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = donorParams->getBuffers()[i];
4907
        extraArgs << ", __global const "+buffer.getType()+"* restrict donor"+buffer.getName();
4908
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, buffer.getType()+" donorParams"+cl.intToString(i+1)+" = donor"+buffer.getName()+"[donorIndex];\n");
4909
4910
4911
    }
    for (int i = 0; i < (int) acceptorParams->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = acceptorParams->getBuffers()[i];
4912
        extraArgs << ", __global const "+buffer.getType()+"* restrict acceptor"+buffer.getName();
4913
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, buffer.getType()+" acceptorParams"+cl.intToString(i+1)+" = acceptor"+buffer.getName()+"[acceptorIndex];\n");
4914
    }
4915
4916
4917

    // Now evaluate the expressions.

4918
    computeAcceptor << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, "temp");
4919
    forceExpressions["energy += "] = energyExpression;
4920
    computeDonor << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, "temp");
4921
4922
4923
4924

    // Finally, apply forces to atoms.

    index = 0;
peastman's avatar
peastman committed
4925
4926
    for (auto& distance : distances) {
        const vector<int>& atoms = distance.second;
4927
        string deltaName = atomNames[atoms[0]]+atomNames[atoms[1]];
4928
        string value = "(dEdDistance"+cl.intToString(index)+"/r_"+deltaName+")*delta"+deltaName+".xyz";
4929
4930
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[0], "-"+value);
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[1], value);
peastman's avatar
peastman committed
4931
        index++;
4932
4933
    }
    index = 0;
peastman's avatar
peastman committed
4934
4935
    for (auto& angle : angles) {
        const vector<int>& atoms = angle.second;
4936
4937
4938
        string deltaName1 = atomNames[atoms[1]]+atomNames[atoms[0]];
        string deltaName2 = atomNames[atoms[1]]+atomNames[atoms[2]];
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "{\n");
4939
4940
4941
4942
4943
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 crossProd = cross(delta"+deltaName2+", delta"+deltaName1+");\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real lengthCross = max(length(crossProd), (real) 1e-6f);\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 deltaCross0 = -cross(delta"+deltaName1+", crossProd)*dEdAngle"+cl.intToString(index)+"/(delta"+deltaName1+".w*lengthCross);\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 deltaCross2 = cross(delta"+deltaName2+", crossProd)*dEdAngle"+cl.intToString(index)+"/(delta"+deltaName2+".w*lengthCross);\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 deltaCross1 = -(deltaCross0+deltaCross2);\n");
4944
4945
4946
4947
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[0], "deltaCross0.xyz");
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[1], "deltaCross1.xyz");
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[2], "deltaCross2.xyz");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "}\n");
peastman's avatar
peastman committed
4948
        index++;
4949
4950
    }
    index = 0;
peastman's avatar
peastman committed
4951
4952
    for (auto& dihedral : dihedrals) {
        const vector<int>& atoms = dihedral.second;
4953
4954
4955
4956
4957
4958
        string deltaName1 = atomNames[atoms[0]]+atomNames[atoms[1]];
        string deltaName2 = atomNames[atoms[2]]+atomNames[atoms[1]];
        string deltaName3 = atomNames[atoms[2]]+atomNames[atoms[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "{\n");
4959
4960
4961
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real r = SQRT(delta"+deltaName2+".w);\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 ff;\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "ff.x = (-dEdDihedral"+cl.intToString(index)+"*r)/"+crossName1+".w;\n");
4962
4963
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "ff.y = (delta"+deltaName1+".x*delta"+deltaName2+".x + delta"+deltaName1+".y*delta"+deltaName2+".y + delta"+deltaName1+".z*delta"+deltaName2+".z)/delta"+deltaName2+".w;\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "ff.z = (delta"+deltaName3+".x*delta"+deltaName2+".x + delta"+deltaName3+".y*delta"+deltaName2+".y + delta"+deltaName3+".z*delta"+deltaName2+".z)/delta"+deltaName2+".w;\n");
4964
4965
4966
4967
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "ff.w = (dEdDihedral"+cl.intToString(index)+"*r)/"+crossName2+".w;\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 internalF0 = ff.x*"+crossName1+";\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 internalF3 = ff.w*"+crossName2+";\n");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "real4 s = ff.y*internalF0 - ff.z*internalF3;\n");
4968
4969
4970
4971
4972
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[0], "internalF0.xyz");
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[1], "s.xyz-internalF0.xyz");
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[2], "-s.xyz-internalF3.xyz");
        applyDonorAndAcceptorForces(computeDonor, computeAcceptor, atoms[3], "internalF3.xyz");
        addDonorAndAcceptorCode(computeDonor, computeAcceptor, "}\n");
peastman's avatar
peastman committed
4973
        index++;
4974
4975
4976
4977
    }

    // Generate the kernels.

4978
    map<string, string> replacements;
4979
4980
    replacements["COMPUTE_DONOR_FORCE"] = computeDonor.str();
    replacements["COMPUTE_ACCEPTOR_FORCE"] = computeAcceptor.str();
4981
4982
    replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
    map<string, string> defines;
4983
4984
4985
4986
    defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
    defines["NUM_DONORS"] = cl.intToString(numDonors);
    defines["NUM_ACCEPTORS"] = cl.intToString(numAcceptors);
    defines["PI"] = cl.doubleToString(M_PI);
4987
4988
    if (force.getNonbondedMethod() != CustomHbondForce::NoCutoff) {
        defines["USE_CUTOFF"] = "1";
4989
        defines["CUTOFF_SQUARED"] = cl.doubleToString(force.getCutoffDistance()*force.getCutoffDistance());
4990
4991
4992
    }
    if (force.getNonbondedMethod() != CustomHbondForce::NoCutoff && force.getNonbondedMethod() != CustomHbondForce::CutoffNonPeriodic)
        defines["USE_PERIODIC"] = "1";
4993
4994
    if (force.getNumExclusions() > 0)
        defines["USE_EXCLUSIONS"] = "1";
4995
    cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customHbondForce, replacements), defines);
4996
4997
    donorKernel = cl::Kernel(program, "computeDonorForces");
    acceptorKernel = cl::Kernel(program, "computeAcceptorForces");
4998
4999
}

5000
double OpenCLCalcCustomHbondForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
5001
5002
    if (numDonors == 0 || numAcceptors == 0)
        return 0.0;
peastman's avatar
peastman committed
5003
    if (globals.isInitialized()) {
5004
5005
5006
5007
5008
5009
5010
5011
        bool changed = false;
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
5012
            globals.upload(globalParamValues);
5013
5014
5015
5016
    }
    if (!hasInitializedKernel) {
        hasInitializedKernel = true;
        int index = 0;
5017
5018
5019
        donorKernel.setArg<cl::Buffer>(index++, cl.getForceBuffers().getDeviceBuffer());
        donorKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
        donorKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
5020
5021
5022
5023
        donorKernel.setArg<cl::Buffer>(index++, donorExclusions.getDeviceBuffer());
        donorKernel.setArg<cl::Buffer>(index++, donors.getDeviceBuffer());
        donorKernel.setArg<cl::Buffer>(index++, acceptors.getDeviceBuffer());
        donorKernel.setArg<cl::Buffer>(index++, donorBufferIndices.getDeviceBuffer());
5024
        donorKernel.setArg(index++, 3*OpenCLContext::ThreadBlockSize*sizeof(mm_float4), NULL);
5025
        index += 5; // Periodic box size arguments are set when the kernel is executed.
peastman's avatar
peastman committed
5026
5027
        if (globals.isInitialized())
            donorKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
5028
        for (auto& buffer : donorParams->getBuffers())
5029
            donorKernel.setArg<cl::Memory>(index++, buffer.getMemory());
peastman's avatar
peastman committed
5030
        for (auto& buffer : acceptorParams->getBuffers())
5031
            donorKernel.setArg<cl::Memory>(index++, buffer.getMemory());
peastman's avatar
peastman committed
5032
5033
        for (auto& function : tabulatedFunctions)
            donorKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
5034
5035
5036
5037
        index = 0;
        acceptorKernel.setArg<cl::Buffer>(index++, cl.getForceBuffers().getDeviceBuffer());
        acceptorKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
        acceptorKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
5038
5039
5040
5041
        acceptorKernel.setArg<cl::Buffer>(index++, acceptorExclusions.getDeviceBuffer());
        acceptorKernel.setArg<cl::Buffer>(index++, donors.getDeviceBuffer());
        acceptorKernel.setArg<cl::Buffer>(index++, acceptors.getDeviceBuffer());
        acceptorKernel.setArg<cl::Buffer>(index++, acceptorBufferIndices.getDeviceBuffer());
5042
        acceptorKernel.setArg(index++, 3*OpenCLContext::ThreadBlockSize*sizeof(mm_float4), NULL);
5043
        index += 5; // Periodic box size arguments are set when the kernel is executed.
peastman's avatar
peastman committed
5044
5045
        if (globals.isInitialized())
            acceptorKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
5046
        for (auto& buffer : donorParams->getBuffers())
5047
            acceptorKernel.setArg<cl::Memory>(index++, buffer.getMemory());
peastman's avatar
peastman committed
5048
        for (auto& buffer : acceptorParams->getBuffers())
5049
            acceptorKernel.setArg<cl::Memory>(index++, buffer.getMemory());
peastman's avatar
peastman committed
5050
5051
        for (auto& function : tabulatedFunctions)
            acceptorKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
5052
    }
5053
    setPeriodicBoxArgs(cl, donorKernel, 8);
Peter Eastman's avatar
Peter Eastman committed
5054
    cl.executeKernel(donorKernel, max(numDonors, numAcceptors));
5055
    setPeriodicBoxArgs(cl, acceptorKernel, 8);
Peter Eastman's avatar
Peter Eastman committed
5056
    cl.executeKernel(acceptorKernel, max(numDonors, numAcceptors));
5057
5058
5059
    return 0.0;
}

5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
void OpenCLCalcCustomHbondForceKernel::copyParametersToContext(ContextImpl& context, const CustomHbondForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumDonors()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumDonors()/numContexts;
    if (numDonors != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of donors has changed");
    if (numAcceptors != force.getNumAcceptors())
        throw OpenMMException("updateParametersInContext: The number of acceptors has changed");
    
    // Record the per-donor parameters.
    
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
    if (numDonors > 0) {
        vector<vector<cl_float> > donorParamVector(numDonors);
        vector<double> parameters;
        for (int i = 0; i < numDonors; i++) {
            int d1, d2, d3;
            force.getDonorParameters(startIndex+i, d1, d2, d3, parameters);
            donorParamVector[i].resize(parameters.size());
            for (int j = 0; j < (int) parameters.size(); j++)
                donorParamVector[i][j] = (cl_float) parameters[j];
        }
        donorParams->setParameterValues(donorParamVector);
5082
5083
5084
5085
    }
    
    // Record the per-acceptor parameters.
    
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
    if (numAcceptors > 0) {
        vector<vector<cl_float> > acceptorParamVector(numAcceptors);
        vector<double> parameters;
        for (int i = 0; i < numAcceptors; i++) {
            int a1, a2, a3;
            force.getAcceptorParameters(i, a1, a2, a3, parameters);
            acceptorParamVector[i].resize(parameters.size());
            for (int j = 0; j < (int) parameters.size(); j++)
                acceptorParamVector[i][j] = (cl_float) parameters[j];
        }
        acceptorParams->setParameterValues(acceptorParamVector);
5097
5098
5099
5100
    }
    
    // Mark that the current reordering may be invalid.
    
5101
    cl.invalidateMolecules(info);
5102
5103
}

5104
class OpenCLCalcCustomCentroidBondForceKernel::ForceInfo : public OpenCLForceInfo {
5105
public:
5106
    ForceInfo(const CustomCentroidBondForce& force) : OpenCLForceInfo(0), force(force) {
5107
5108
5109
5110
5111
5112
5113
5114
    }
    int getNumParticleGroups() {
        return force.getNumBonds();
    }
    void getParticlesInGroup(int index, vector<int>& particles) {
        vector<double> parameters;
        vector<int> groups;
        force.getBondParameters(index, groups, parameters);
peastman's avatar
peastman committed
5115
        for (int group : groups) {
5116
5117
            vector<int> groupParticles;
            vector<double> weights;
peastman's avatar
peastman committed
5118
            force.getGroupParameters(group, groupParticles, weights);
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
            particles.insert(particles.end(), groupParticles.begin(), groupParticles.end());
        }
    }
    bool areGroupsIdentical(int group1, int group2) {
        vector<int> groups1, groups2;
        vector<double> parameters1, parameters2;
        force.getBondParameters(group1, groups1, parameters1);
        force.getBondParameters(group2, groups2, parameters2);
        for (int i = 0; i < (int) parameters1.size(); i++)
            if (parameters1[i] != parameters2[i])
                return false;
        for (int i = 0; i < groups1.size(); i++) {
            vector<int> groupParticles;
            vector<double> weights1, weights2;
            force.getGroupParameters(groups1[i], groupParticles, weights1);
            force.getGroupParameters(groups2[i], groupParticles, weights2);
            if (weights1.size() != weights2.size())
                return false;
            for (int j = 0; j < weights1.size(); j++)
                if (weights1[j] != weights2[j])
                    return false;
        }
        return true;
    }
private:
    const CustomCentroidBondForce& force;
};

OpenCLCalcCustomCentroidBondForceKernel::~OpenCLCalcCustomCentroidBondForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomCentroidBondForceKernel::initialize(const System& system, const CustomCentroidBondForce& force) {
    numBonds = force.getNumBonds();
    if (numBonds == 0)
        return;
    if (!cl.getSupports64BitGlobalAtomics())
        throw OpenMMException("CustomCentroidBondForce requires a device that supports 64 bit atomic operations");
5158
5159
    info = new ForceInfo(force);
    cl.addForce(info);
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
    
    // Record the groups.
    
    numGroups = force.getNumGroups();
    vector<cl_int> groupParticleVec;
    vector<cl_float> groupWeightVecFloat;
    vector<cl_double> groupWeightVecDouble;
    vector<cl_int> groupOffsetVec;
    groupOffsetVec.push_back(0);
    for (int i = 0; i < numGroups; i++) {
        vector<int> particles;
        vector<double> weights;
        force.getGroupParameters(i, particles, weights);
        groupParticleVec.insert(groupParticleVec.end(), particles.begin(), particles.end());
        groupOffsetVec.push_back(groupParticleVec.size());
    }
    vector<vector<double> > normalizedWeights;
    CustomCentroidBondForceImpl::computeNormalizedWeights(force, system, normalizedWeights);
    if (cl.getUseDoublePrecision()) {
        for (int i = 0; i < numGroups; i++)
            groupWeightVecDouble.insert(groupWeightVecDouble.end(), normalizedWeights[i].begin(), normalizedWeights[i].end());
    }
    else {
        for (int i = 0; i < numGroups; i++)
            for (int j = 0; j < normalizedWeights[i].size(); j++)
                groupWeightVecFloat.push_back((float) normalizedWeights[i][j]);
    }
peastman's avatar
peastman committed
5187
5188
    groupParticles.initialize<int>(cl, groupParticleVec.size(), "groupParticles");
    groupParticles.upload(groupParticleVec);
5189
    if (cl.getUseDoublePrecision()) {
peastman's avatar
peastman committed
5190
5191
5192
        groupWeights.initialize<double>(cl, groupParticleVec.size(), "groupWeights");
        groupWeights.upload(groupWeightVecDouble);
        centerPositions.initialize<mm_double4>(cl, numGroups, "centerPositions");
5193
5194
    }
    else {
peastman's avatar
peastman committed
5195
5196
5197
5198
5199
5200
5201
5202
        groupWeights.initialize<float>(cl, groupParticleVec.size(), "groupWeights");
        groupWeights.upload(groupWeightVecFloat);
        centerPositions.initialize<mm_float4>(cl, numGroups, "centerPositions");
    }
    groupOffsets.initialize<int>(cl, groupOffsetVec.size(), "groupOffsets");
    groupOffsets.upload(groupOffsetVec);
    groupForces.initialize<long long>(cl, numGroups*3, "groupForces");
    cl.addAutoclearBuffer(groupForces);
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
    
    // Record the bonds.
    
    int groupsPerBond = force.getNumGroupsPerBond();
    vector<cl_int> bondGroupVec(numBonds*groupsPerBond);
    params = new OpenCLParameterSet(cl, force.getNumPerBondParameters(), numBonds, "customCentroidBondParams");
    vector<vector<float> > paramVector(numBonds);
    for (int i = 0; i < numBonds; i++) {
        vector<int> groups;
        vector<double> parameters;
        force.getBondParameters(i, groups, parameters);
        for (int j = 0; j < groups.size(); j++)
            bondGroupVec[i+j*numBonds] = groups[j];
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (float) parameters[j];
    }
    params->setParameterValues(paramVector);
peastman's avatar
peastman committed
5221
5222
    bondGroups.initialize<int>(cl, bondGroupVec.size(), "bondGroups");
    bondGroups.upload(bondGroupVec);
5223
5224
5225
5226
5227
5228
5229

    // Record the tabulated functions.

    map<string, Lepton::CustomFunction*> functions;
    vector<pair<string, string> > functionDefinitions;
    vector<const TabulatedFunction*> functionList;
    stringstream extraArgs;
peastman's avatar
peastman committed
5230
    tabulatedFunctions.resize(force.getNumTabulatedFunctions());
5231
5232
5233
5234
5235
5236
5237
5238
    for (int i = 0; i < force.getNumTabulatedFunctions(); i++) {
        functionList.push_back(&force.getTabulatedFunction(i));
        string name = force.getTabulatedFunctionName(i);
        string arrayName = "table"+cl.intToString(i);
        functionDefinitions.push_back(make_pair(name, arrayName));
        functions[name] = cl.getExpressionUtilities().getFunctionPlaceholder(force.getTabulatedFunction(i));
        int width;
        vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(force.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
5239
5240
        tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
        tabulatedFunctions[i].upload(f);
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
        extraArgs << ", __global const float";
        if (width > 1)
            extraArgs << width;
        extraArgs << "* restrict " << arrayName;
    }
    
    // Record information about parameters.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (float) force.getGlobalParameterDefaultValue(i);
    }
    map<string, string> variables;
    for (int i = 0; i < groupsPerBond; i++) {
        string index = cl.intToString(i+1);
        variables["x"+index] = "pos"+index+".x";
        variables["y"+index] = "pos"+index+".y";
        variables["z"+index] = "pos"+index+".z";
    }
    for (int i = 0; i < force.getNumPerBondParameters(); i++) {
        const string& name = force.getPerBondParameterName(i);
        variables[name] = "bondParams"+params->getParameterSuffix(i);
    }
5266
5267
5268
    needEnergyParamDerivs = (force.getNumEnergyParameterDerivatives() > 0);
    if (needEnergyParamDerivs)
        extraArgs << ", __global mixed* restrict energyParamDerivs";
5269
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
5270
5271
        globals.initialize<float>(cl, force.getNumGlobalParameters(), "customCentroidBondGlobals");
        globals.upload(globalParamValues);
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
        extraArgs << ", __global const float* restrict globals";
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
            string value = "globals["+cl.intToString(i)+"]";
            variables[name] = value;
        }
    }

    // Now to generate the kernel.  First, it needs to calculate all distances, angles,
    // and dihedrals the expression depends on.

    map<string, vector<int> > distances;
    map<string, vector<int> > angles;
    map<string, vector<int> > dihedrals;
    Lepton::ParsedExpression energyExpression = CustomCentroidBondForceImpl::prepareExpression(force, functions, distances, angles, dihedrals);
    map<string, Lepton::ParsedExpression> forceExpressions;
    set<string> computedDeltas;
    vector<string> atomNames, posNames;
    for (int i = 0; i < groupsPerBond; i++) {
        string index = cl.intToString(i+1);
        atomNames.push_back("P"+index);
        posNames.push_back("pos"+index);
    }
5295
    stringstream compute, initParamDerivs, saveParamDerivs;
5296
5297
5298
5299
5300
    for (int i = 0; i < groupsPerBond; i++) {
        compute<<"int group"<<(i+1)<<" = bondGroups[index+"<<(i*numBonds)<<"];\n";
        compute<<"real4 pos"<<(i+1)<<" = centerPositions[group"<<(i+1)<<"];\n";
    }
    int index = 0;
peastman's avatar
peastman committed
5301
5302
    for (auto& distance : distances) {
        const vector<int>& groups = distance.second;
5303
5304
        string deltaName = atomNames[groups[0]]+atomNames[groups[1]];
        if (computedDeltas.count(deltaName) == 0) {
5305
            compute<<"real4 delta"<<deltaName<<" = delta("<<posNames[groups[0]]<<", "<<posNames[groups[1]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5306
5307
5308
            computedDeltas.insert(deltaName);
        }
        compute<<"real r_"<<deltaName<<" = sqrt(delta"<<deltaName<<".w);\n";
peastman's avatar
peastman committed
5309
5310
5311
        variables[distance.first] = "r_"+deltaName;
        forceExpressions["real dEdDistance"+cl.intToString(index)+" = "] = energyExpression.differentiate(distance.first).optimize();
        index++;
5312
5313
    }
    index = 0;
peastman's avatar
peastman committed
5314
5315
    for (auto& angle : angles) {
        const vector<int>& groups = angle.second;
5316
5317
5318
5319
        string deltaName1 = atomNames[groups[1]]+atomNames[groups[0]];
        string deltaName2 = atomNames[groups[1]]+atomNames[groups[2]];
        string angleName = "angle_"+atomNames[groups[0]]+atomNames[groups[1]]+atomNames[groups[2]];
        if (computedDeltas.count(deltaName1) == 0) {
5320
            compute<<"real4 delta"<<deltaName1<<" = delta("<<posNames[groups[1]]<<", "<<posNames[groups[0]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5321
5322
5323
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
5324
            compute<<"real4 delta"<<deltaName2<<" = delta("<<posNames[groups[1]]<<", "<<posNames[groups[2]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5325
5326
5327
            computedDeltas.insert(deltaName2);
        }
        compute<<"real "<<angleName<<" = computeAngle(delta"<<deltaName1<<", delta"<<deltaName2<<");\n";
peastman's avatar
peastman committed
5328
5329
5330
        variables[angle.first] = angleName;
        forceExpressions["real dEdAngle"+cl.intToString(index)+" = "] = energyExpression.differentiate(angle.first).optimize();
        index++;
5331
5332
    }
    index = 0;
peastman's avatar
peastman committed
5333
5334
    for (auto& dihedral : dihedrals) {
        const vector<int>& groups = dihedral.second;
5335
5336
5337
5338
5339
5340
5341
        string deltaName1 = atomNames[groups[0]]+atomNames[groups[1]];
        string deltaName2 = atomNames[groups[2]]+atomNames[groups[1]];
        string deltaName3 = atomNames[groups[2]]+atomNames[groups[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        string dihedralName = "dihedral_"+atomNames[groups[0]]+atomNames[groups[1]]+atomNames[groups[2]]+atomNames[groups[3]];
        if (computedDeltas.count(deltaName1) == 0) {
5342
            compute<<"real4 delta"<<deltaName1<<" = delta("<<posNames[groups[0]]<<", "<<posNames[groups[1]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5343
5344
5345
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
5346
            compute<<"real4 delta"<<deltaName2<<" = delta("<<posNames[groups[2]]<<", "<<posNames[groups[1]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5347
5348
5349
            computedDeltas.insert(deltaName2);
        }
        if (computedDeltas.count(deltaName3) == 0) {
5350
            compute<<"real4 delta"<<deltaName3<<" = delta("<<posNames[groups[2]]<<", "<<posNames[groups[3]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5351
5352
5353
5354
5355
5356
            computedDeltas.insert(deltaName3);
        }
        compute<<"real4 "<<crossName1<<" = computeCross(delta"<<deltaName1<<", delta"<<deltaName2<<");\n";
        compute<<"real4 "<<crossName2<<" = computeCross(delta"<<deltaName2<<", delta"<<deltaName3<<");\n";
        compute<<"real "<<dihedralName<<" = computeAngle("<<crossName1<<", "<<crossName2<<");\n";
        compute<<dihedralName<<" *= (delta"<<deltaName1<<".x*"<<crossName2<<".x + delta"<<deltaName1<<".y*"<<crossName2<<".y + delta"<<deltaName1<<".z*"<<crossName2<<".z < 0 ? -1 : 1);\n";
peastman's avatar
peastman committed
5357
5358
5359
        variables[dihedral.first] = dihedralName;
        forceExpressions["real dEdDihedral"+cl.intToString(index)+" = "] = energyExpression.differentiate(dihedral.first).optimize();
        index++;
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
    }

    // Now evaluate the expressions.

    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
        extraArgs<<", __global const "<<buffer.getType()<<"* restrict globalParams"<<i;
        compute<<buffer.getType()<<" bondParams"<<(i+1)<<" = globalParams"<<i<<"[index];\n";
    }
    forceExpressions["energy += "] = energyExpression;
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
    if (needEnergyParamDerivs) {
        for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
            string paramName = force.getEnergyParameterDerivativeName(i);
            cl.addEnergyParameterDerivative(paramName);
            Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
            forceExpressions[string("energyParamDeriv")+cl.intToString(i)+" += "] = derivExpression;
            initParamDerivs << "mixed energyParamDeriv" << i << " = 0;\n";
        }
        const vector<string>& allParamDerivNames = cl.getEnergyParamDerivNames();
        int numDerivs = allParamDerivNames.size();
        for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++)
            for (int index = 0; index < numDerivs; index++)
                if (allParamDerivNames[index] == force.getEnergyParameterDerivativeName(i))
                    saveParamDerivs << "energyParamDerivs[get_global_id(0)*" << numDerivs << "+" << index << "] += energyParamDeriv" << i << ";\n";
    }
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
    compute << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, "temp");

    // Finally, apply forces to groups.

    vector<string> forceNames;
    for (int i = 0; i < groupsPerBond; i++) {
        string istr = cl.intToString(i+1);
        string forceName = "force"+istr;
        forceNames.push_back(forceName);
        compute<<"real3 "<<forceName<<" = (real3) 0;\n";
        compute<<"{\n";
        Lepton::ParsedExpression forceExpressionX = energyExpression.differentiate("x"+istr).optimize();
        Lepton::ParsedExpression forceExpressionY = energyExpression.differentiate("y"+istr).optimize();
        Lepton::ParsedExpression forceExpressionZ = energyExpression.differentiate("z"+istr).optimize();
        map<string, Lepton::ParsedExpression> expressions;
        if (!isZeroExpression(forceExpressionX))
            expressions[forceName+".x -= "] = forceExpressionX;
        if (!isZeroExpression(forceExpressionY))
            expressions[forceName+".y -= "] = forceExpressionY;
        if (!isZeroExpression(forceExpressionZ))
            expressions[forceName+".z -= "] = forceExpressionZ;
        if (expressions.size() > 0)
            compute<<cl.getExpressionUtilities().createExpressions(expressions, variables, functionList, functionDefinitions, "coordtemp");
        compute<<"}\n";
    }
    index = 0;
peastman's avatar
peastman committed
5411
5412
    for (auto& distance : distances) {
        const vector<int>& groups = distance.second;
5413
5414
5415
5416
        string deltaName = atomNames[groups[0]]+atomNames[groups[1]];
        string value = "(dEdDistance"+cl.intToString(index)+"/r_"+deltaName+")*delta"+deltaName+".xyz";
        compute<<forceNames[groups[0]]<<" += "<<"-"<<value<<";\n";
        compute<<forceNames[groups[1]]<<" += "<<value<<";\n";
peastman's avatar
peastman committed
5417
        index++;
5418
5419
    }
    index = 0;
peastman's avatar
peastman committed
5420
5421
    for (auto& angle : angles) {
        const vector<int>& groups = angle.second;
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
        string deltaName1 = atomNames[groups[1]]+atomNames[groups[0]];
        string deltaName2 = atomNames[groups[1]]+atomNames[groups[2]];
        compute<<"{\n";
        compute<<"real4 crossProd = cross(delta"<<deltaName2<<", delta"<<deltaName1<<");\n";
        compute<<"real lengthCross = max(length(crossProd), (real) 1e-6f);\n";
        compute<<"real4 deltaCross0 = -cross(delta"<<deltaName1<<", crossProd)*dEdAngle"<<cl.intToString(index)<<"/(delta"<<deltaName1<<".w*lengthCross);\n";
        compute<<"real4 deltaCross2 = cross(delta"<<deltaName2<<", crossProd)*dEdAngle"<<cl.intToString(index)<<"/(delta"<<deltaName2<<".w*lengthCross);\n";
        compute<<"real4 deltaCross1 = -(deltaCross0+deltaCross2);\n";
        compute<<forceNames[groups[0]]<<".xyz += deltaCross0.xyz;\n";
        compute<<forceNames[groups[1]]<<".xyz += deltaCross1.xyz;\n";
        compute<<forceNames[groups[2]]<<".xyz += deltaCross2.xyz;\n";
        compute<<"}\n";
peastman's avatar
peastman committed
5434
        index++;
5435
5436
    }
    index = 0;
peastman's avatar
peastman committed
5437
5438
    for (auto& dihedral : dihedrals) {
        const vector<int>& groups = dihedral.second;
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
        string deltaName1 = atomNames[groups[0]]+atomNames[groups[1]];
        string deltaName2 = atomNames[groups[2]]+atomNames[groups[1]];
        string deltaName3 = atomNames[groups[2]]+atomNames[groups[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        compute<<"{\n";
        compute<<"real r = sqrt(delta"<<deltaName2<<".w);\n";
        compute<<"real4 ff;\n";
        compute<<"ff.x = (-dEdDihedral"<<cl.intToString(index)<<"*r)/"<<crossName1<<".w;\n";
        compute<<"ff.y = (delta"<<deltaName1<<".x*delta"<<deltaName2<<".x + delta"<<deltaName1<<".y*delta"<<deltaName2<<".y + delta"<<deltaName1<<".z*delta"<<deltaName2<<".z)/delta"<<deltaName2<<".w;\n";
        compute<<"ff.z = (delta"<<deltaName3<<".x*delta"<<deltaName2<<".x + delta"<<deltaName3<<".y*delta"<<deltaName2<<".y + delta"<<deltaName3<<".z*delta"<<deltaName2<<".z)/delta"<<deltaName2<<".w;\n";
        compute<<"ff.w = (dEdDihedral"<<cl.intToString(index)<<"*r)/"<<crossName2<<".w;\n";
        compute<<"real4 internalF0 = ff.x*"<<crossName1<<";\n";
        compute<<"real4 internalF3 = ff.w*"<<crossName2<<";\n";
        compute<<"real4 s = ff.y*internalF0 - ff.z*internalF3;\n";
        compute<<forceNames[groups[0]]<<".xyz += internalF0.xyz;\n";
        compute<<forceNames[groups[1]]<<".xyz += s.xyz-internalF0.xyz;\n";
        compute<<forceNames[groups[2]]<<".xyz += -s.xyz-internalF3.xyz;\n";
        compute<<forceNames[groups[3]]<<".xyz += internalF3.xyz;\n";
        compute<<"}\n";
peastman's avatar
peastman committed
5459
        index++;
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
    }
    
    // Save the forces to global memory.
    
    for (int i = 0; i < groupsPerBond; i++) {
        compute<<"atom_add(&groupForce[group"<<(i+1)<<"], (long) (force"<<(i+1)<<".x*0x100000000));\n";
        compute<<"atom_add(&groupForce[group"<<(i+1)<<"+NUM_GROUPS], (long) (force"<<(i+1)<<".y*0x100000000));\n";
        compute<<"atom_add(&groupForce[group"<<(i+1)<<"+NUM_GROUPS*2], (long) (force"<<(i+1)<<".z*0x100000000));\n";
    }
    map<string, string> replacements;
    replacements["M_PI"] = cl.doubleToString(M_PI);
    replacements["NUM_GROUPS"] = cl.intToString(numGroups);
    replacements["NUM_BONDS"] = cl.intToString(numBonds);
    replacements["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
    replacements["EXTRA_ARGS"] = extraArgs.str();
    replacements["COMPUTE_FORCE"] = compute.str();
5476
5477
    replacements["INIT_PARAM_DERIVS"] = initParamDerivs.str();
    replacements["SAVE_PARAM_DERIVS"] = saveParamDerivs.str();
5478
5479
5480
5481
    cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customCentroidBond, replacements));
    index = 0;
    computeCentersKernel = cl::Kernel(program, "computeGroupCenters");
    computeCentersKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
5482
5483
5484
5485
    computeCentersKernel.setArg<cl::Buffer>(index++, groupParticles.getDeviceBuffer());
    computeCentersKernel.setArg<cl::Buffer>(index++, groupWeights.getDeviceBuffer());
    computeCentersKernel.setArg<cl::Buffer>(index++, groupOffsets.getDeviceBuffer());
    computeCentersKernel.setArg<cl::Buffer>(index++, centerPositions.getDeviceBuffer());
5486
5487
    index = 0;
    groupForcesKernel = cl::Kernel(program, "computeGroupForces");
peastman's avatar
peastman committed
5488
    groupForcesKernel.setArg<cl::Buffer>(index++, groupForces.getDeviceBuffer());
5489
    index++; // Energy buffer hasn't been created yet
peastman's avatar
peastman committed
5490
5491
    groupForcesKernel.setArg<cl::Buffer>(index++, centerPositions.getDeviceBuffer());
    groupForcesKernel.setArg<cl::Buffer>(index++, bondGroups.getDeviceBuffer());
5492
    index += 5; // Periodic box information
5493
5494
    if (needEnergyParamDerivs)
        index++; // Deriv buffer hasn't been created yet.
peastman's avatar
peastman committed
5495
5496
5497
5498
    for (auto& function : tabulatedFunctions)
        groupForcesKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
    if (globals.isInitialized())
        groupForcesKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
5499
5500
    for (auto& buffer : params->getBuffers())
        groupForcesKernel.setArg<cl::Memory>(index++, buffer.getMemory());
5501
5502
    index = 0;
    applyForcesKernel = cl::Kernel(program, "applyForcesToAtoms");
peastman's avatar
peastman committed
5503
5504
5505
5506
    applyForcesKernel.setArg<cl::Buffer>(index++, groupParticles.getDeviceBuffer());
    applyForcesKernel.setArg<cl::Buffer>(index++, groupWeights.getDeviceBuffer());
    applyForcesKernel.setArg<cl::Buffer>(index++, groupOffsets.getDeviceBuffer());
    applyForcesKernel.setArg<cl::Buffer>(index++, groupForces.getDeviceBuffer());
5507
5508
5509
}

double OpenCLCalcCustomCentroidBondForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
5510
5511
    if (numBonds == 0)
        return 0.0;
peastman's avatar
peastman committed
5512
    if (globals.isInitialized()) {
5513
5514
5515
5516
5517
5518
5519
5520
        bool changed = false;
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
            float value = (float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
5521
            globals.upload(globalParamValues);
5522
5523
5524
    }
    cl.executeKernel(computeCentersKernel, OpenCLContext::TileSize*numGroups);
    groupForcesKernel.setArg<cl::Buffer>(1, cl.getEnergyBuffer().getDeviceBuffer());
5525
    setPeriodicBoxArgs(cl, groupForcesKernel, 4);
5526
5527
    if (needEnergyParamDerivs)
        groupForcesKernel.setArg<cl::Memory>(9, cl.getEnergyParamDerivBuffer().getDeviceBuffer());
5528
5529
5530
5531
5532
5533
5534
    cl.executeKernel(groupForcesKernel, numBonds);
    applyForcesKernel.setArg<cl::Buffer>(4, cl.getLongForceBuffer().getDeviceBuffer());
    cl.executeKernel(applyForcesKernel, OpenCLContext::TileSize*numGroups);
    return 0.0;
}

void OpenCLCalcCustomCentroidBondForceKernel::copyParametersToContext(ContextImpl& context, const CustomCentroidBondForce& force) {
5535
    if (numBonds != force.getNumBonds())
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
        throw OpenMMException("updateParametersInContext: The number of bonds has changed");
    if (numBonds == 0)
        return;
    
    // Record the per-bond parameters.
    
    vector<vector<float> > paramVector(numBonds);
    vector<int> particles;
    vector<double> parameters;
    for (int i = 0; i < numBonds; i++) {
5546
        force.getBondParameters(i, particles, parameters);
5547
5548
5549
5550
5551
5552
5553
5554
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
5555
    cl.invalidateMolecules(info);
5556
5557
}

5558
class OpenCLCalcCustomCompoundBondForceKernel::ForceInfo : public OpenCLForceInfo {
5559
public:
5560
    ForceInfo(const CustomCompoundBondForce& force) : OpenCLForceInfo(0), force(force) {
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
    }
    int getNumParticleGroups() {
        return force.getNumBonds();
    }
    void getParticlesInGroup(int index, vector<int>& particles) {
        vector<double> parameters;
        force.getBondParameters(index, particles, parameters);
    }
    bool areGroupsIdentical(int group1, int group2) {
        vector<int> particles;
        vector<double> parameters1, parameters2;
        force.getBondParameters(group1, particles, parameters1);
        force.getBondParameters(group2, particles, parameters2);
        for (int i = 0; i < (int) parameters1.size(); i++)
            if (parameters1[i] != parameters2[i])
                return false;
        return true;
    }
private:
    const CustomCompoundBondForce& force;
};

OpenCLCalcCustomCompoundBondForceKernel::~OpenCLCalcCustomCompoundBondForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomCompoundBondForceKernel::initialize(const System& system, const CustomCompoundBondForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumBonds()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumBonds()/numContexts;
    numBonds = endIndex-startIndex;
    if (numBonds == 0)
        return;
    int particlesPerBond = force.getNumParticlesPerBond();
    vector<vector<int> > atoms(numBonds, vector<int>(particlesPerBond));
    params = new OpenCLParameterSet(cl, force.getNumPerBondParameters(), numBonds, "customCompoundBondParams");
    vector<vector<cl_float> > paramVector(numBonds);
    for (int i = 0; i < numBonds; i++) {
        vector<double> parameters;
        force.getBondParameters(startIndex+i, atoms[i], parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
5607
5608
    info = new ForceInfo(force);
    cl.addForce(info);
5609
5610
5611
5612
5613

    // Record the tabulated functions.

    map<string, Lepton::CustomFunction*> functions;
    vector<pair<string, string> > functionDefinitions;
5614
    vector<const TabulatedFunction*> functionList;
5615
    stringstream tableArgs;
peastman's avatar
peastman committed
5616
5617
    tabulatedFunctions.resize(force.getNumTabulatedFunctions());
    for (int i = 0; i < force.getNumTabulatedFunctions(); i++) {
5618
5619
5620
        functionList.push_back(&force.getTabulatedFunction(i));
        string name = force.getTabulatedFunctionName(i);
        functions[name] = cl.getExpressionUtilities().getFunctionPlaceholder(force.getTabulatedFunction(i));
peastman's avatar
peastman committed
5621
        int width;
5622
        vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(force.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
5623
5624
5625
        tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
        tabulatedFunctions[i].upload(f);
        string arrayName = cl.getBondedUtilities().addArgument(tabulatedFunctions[i].getDeviceBuffer(), width == 1 ? "float" : "float"+cl.intToString(width));
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
        functionDefinitions.push_back(make_pair(name, arrayName));
    }
    
    // Record information about parameters.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (cl_float) force.getGlobalParameterDefaultValue(i);
    }
    map<string, string> variables;
    for (int i = 0; i < particlesPerBond; i++) {
5639
        string index = cl.intToString(i+1);
5640
5641
5642
5643
5644
5645
5646
5647
5648
        variables["x"+index] = "pos"+index+".x";
        variables["y"+index] = "pos"+index+".y";
        variables["z"+index] = "pos"+index+".z";
    }
    for (int i = 0; i < force.getNumPerBondParameters(); i++) {
        const string& name = force.getPerBondParameterName(i);
        variables[name] = "bondParams"+params->getParameterSuffix(i);
    }
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
5649
5650
5651
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customCompoundBondGlobals", CL_MEM_READ_ONLY);
        globals.upload(globalParamValues);
        string argName = cl.getBondedUtilities().addArgument(globals.getDeviceBuffer(), "float");
5652
5653
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
5654
            string value = argName+"["+cl.intToString(i)+"]";
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
            variables[name] = value;
        }
    }

    // Now to generate the kernel.  First, it needs to calculate all distances, angles,
    // and dihedrals the expression depends on.

    map<string, vector<int> > distances;
    map<string, vector<int> > angles;
    map<string, vector<int> > dihedrals;
    Lepton::ParsedExpression energyExpression = CustomCompoundBondForceImpl::prepareExpression(force, functions, distances, angles, dihedrals);
    map<string, Lepton::ParsedExpression> forceExpressions;
    set<string> computedDeltas;
    vector<string> atomNames, posNames;
    for (int i = 0; i < particlesPerBond; i++) {
5670
        string index = cl.intToString(i+1);
5671
5672
5673
5674
5675
        atomNames.push_back("P"+index);
        posNames.push_back("pos"+index);
    }
    stringstream compute;
    int index = 0;
peastman's avatar
peastman committed
5676
5677
    for (auto& distance : distances) {
        const vector<int>& atoms = distance.second;
5678
5679
        string deltaName = atomNames[atoms[0]]+atomNames[atoms[1]];
        if (computedDeltas.count(deltaName) == 0) {
5680
            compute<<"real4 delta"<<deltaName<<" = ccb_delta("<<posNames[atoms[0]]<<", "<<posNames[atoms[1]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5681
5682
            computedDeltas.insert(deltaName);
        }
5683
        compute<<"real r_"<<deltaName<<" = sqrt(delta"<<deltaName<<".w);\n";
peastman's avatar
peastman committed
5684
5685
5686
        variables[distance.first] = "r_"+deltaName;
        forceExpressions["real dEdDistance"+cl.intToString(index)+" = "] = energyExpression.differentiate(distance.first).optimize();
        index++;
5687
5688
    }
    index = 0;
peastman's avatar
peastman committed
5689
5690
    for (auto& angle : angles) {
        const vector<int>& atoms = angle.second;
5691
5692
5693
5694
        string deltaName1 = atomNames[atoms[1]]+atomNames[atoms[0]];
        string deltaName2 = atomNames[atoms[1]]+atomNames[atoms[2]];
        string angleName = "angle_"+atomNames[atoms[0]]+atomNames[atoms[1]]+atomNames[atoms[2]];
        if (computedDeltas.count(deltaName1) == 0) {
5695
            compute<<"real4 delta"<<deltaName1<<" = ccb_delta("<<posNames[atoms[1]]<<", "<<posNames[atoms[0]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5696
5697
5698
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
5699
            compute<<"real4 delta"<<deltaName2<<" = ccb_delta("<<posNames[atoms[1]]<<", "<<posNames[atoms[2]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5700
5701
            computedDeltas.insert(deltaName2);
        }
5702
        compute<<"real "<<angleName<<" = ccb_computeAngle(delta"<<deltaName1<<", delta"<<deltaName2<<");\n";
peastman's avatar
peastman committed
5703
5704
5705
        variables[angle.first] = angleName;
        forceExpressions["real dEdAngle"+cl.intToString(index)+" = "] = energyExpression.differentiate(angle.first).optimize();
        index++;
5706
5707
    }
    index = 0;
peastman's avatar
peastman committed
5708
5709
    for (auto& dihedral : dihedrals) {
        const vector<int>& atoms = dihedral.second;
5710
5711
5712
5713
5714
5715
5716
        string deltaName1 = atomNames[atoms[0]]+atomNames[atoms[1]];
        string deltaName2 = atomNames[atoms[2]]+atomNames[atoms[1]];
        string deltaName3 = atomNames[atoms[2]]+atomNames[atoms[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        string dihedralName = "dihedral_"+atomNames[atoms[0]]+atomNames[atoms[1]]+atomNames[atoms[2]]+atomNames[atoms[3]];
        if (computedDeltas.count(deltaName1) == 0) {
5717
            compute<<"real4 delta"<<deltaName1<<" = ccb_delta("<<posNames[atoms[0]]<<", "<<posNames[atoms[1]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5718
5719
5720
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
5721
            compute<<"real4 delta"<<deltaName2<<" = ccb_delta("<<posNames[atoms[2]]<<", "<<posNames[atoms[1]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5722
5723
5724
            computedDeltas.insert(deltaName2);
        }
        if (computedDeltas.count(deltaName3) == 0) {
5725
            compute<<"real4 delta"<<deltaName3<<" = ccb_delta("<<posNames[atoms[2]]<<", "<<posNames[atoms[3]]<<", "<<force.usesPeriodicBoundaryConditions()<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
5726
5727
            computedDeltas.insert(deltaName3);
        }
5728
5729
5730
        compute<<"real4 "<<crossName1<<" = ccb_computeCross(delta"<<deltaName1<<", delta"<<deltaName2<<");\n";
        compute<<"real4 "<<crossName2<<" = ccb_computeCross(delta"<<deltaName2<<", delta"<<deltaName3<<");\n";
        compute<<"real "<<dihedralName<<" = ccb_computeAngle("<<crossName1<<", "<<crossName2<<");\n";
5731
        compute<<dihedralName<<" *= (delta"<<deltaName1<<".x*"<<crossName2<<".x + delta"<<deltaName1<<".y*"<<crossName2<<".y + delta"<<deltaName1<<".z*"<<crossName2<<".z < 0 ? -1 : 1);\n";
peastman's avatar
peastman committed
5732
5733
5734
        variables[dihedral.first] = dihedralName;
        forceExpressions["real dEdDihedral"+cl.intToString(index)+" = "] = energyExpression.differentiate(dihedral.first).optimize();
        index++;
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
    }

    // Now evaluate the expressions.

    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        const OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
        string argName = cl.getBondedUtilities().addArgument(buffer.getMemory(), buffer.getType());
        compute<<buffer.getType()<<" bondParams"<<(i+1)<<" = "<<argName<<"[index];\n";
    }
    forceExpressions["energy += "] = energyExpression;
5745
5746
5747
5748
5749
5750
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string paramName = force.getEnergyParameterDerivativeName(i);
        string derivVariable = cl.getBondedUtilities().addEnergyParameterDerivative(paramName);
        Lepton::ParsedExpression derivExpression = energyExpression.differentiate(paramName).optimize();
        forceExpressions[derivVariable+" += "] = derivExpression;
    }
5751
    compute << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, "temp");
5752
5753
5754
5755
5756

    // Finally, apply forces to atoms.

    vector<string> forceNames;
    for (int i = 0; i < particlesPerBond; i++) {
5757
        string istr = cl.intToString(i+1);
5758
5759
        string forceName = "force"+istr;
        forceNames.push_back(forceName);
5760
        compute<<"real4 "<<forceName<<" = (real4) 0;\n";
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
        compute<<"{\n";
        Lepton::ParsedExpression forceExpressionX = energyExpression.differentiate("x"+istr).optimize();
        Lepton::ParsedExpression forceExpressionY = energyExpression.differentiate("y"+istr).optimize();
        Lepton::ParsedExpression forceExpressionZ = energyExpression.differentiate("z"+istr).optimize();
        map<string, Lepton::ParsedExpression> expressions;
        if (!isZeroExpression(forceExpressionX))
            expressions[forceName+".x -= "] = forceExpressionX;
        if (!isZeroExpression(forceExpressionY))
            expressions[forceName+".y -= "] = forceExpressionY;
        if (!isZeroExpression(forceExpressionZ))
            expressions[forceName+".z -= "] = forceExpressionZ;
        if (expressions.size() > 0)
5773
            compute<<cl.getExpressionUtilities().createExpressions(expressions, variables, functionList, functionDefinitions, "coordtemp");
5774
5775
5776
        compute<<"}\n";
    }
    index = 0;
peastman's avatar
peastman committed
5777
5778
    for (auto& distance : distances) {
        const vector<int>& atoms = distance.second;
5779
        string deltaName = atomNames[atoms[0]]+atomNames[atoms[1]];
5780
        string value = "(dEdDistance"+cl.intToString(index)+"/r_"+deltaName+")*delta"+deltaName+".xyz";
5781
5782
        compute<<forceNames[atoms[0]]<<".xyz += "<<"-"<<value<<";\n";
        compute<<forceNames[atoms[1]]<<".xyz += "<<value<<";\n";
peastman's avatar
peastman committed
5783
        index++;
5784
5785
    }
    index = 0;
peastman's avatar
peastman committed
5786
5787
    for (auto& angle : angles) {
        const vector<int>& atoms = angle.second;
5788
5789
5790
        string deltaName1 = atomNames[atoms[1]]+atomNames[atoms[0]];
        string deltaName2 = atomNames[atoms[1]]+atomNames[atoms[2]];
        compute<<"{\n";
5791
5792
5793
5794
5795
        compute<<"real4 crossProd = cross(delta"<<deltaName2<<", delta"<<deltaName1<<");\n";
        compute<<"real lengthCross = max(length(crossProd), (real) 1e-6f);\n";
        compute<<"real4 deltaCross0 = -cross(delta"<<deltaName1<<", crossProd)*dEdAngle"<<cl.intToString(index)<<"/(delta"<<deltaName1<<".w*lengthCross);\n";
        compute<<"real4 deltaCross2 = cross(delta"<<deltaName2<<", crossProd)*dEdAngle"<<cl.intToString(index)<<"/(delta"<<deltaName2<<".w*lengthCross);\n";
        compute<<"real4 deltaCross1 = -(deltaCross0+deltaCross2);\n";
5796
5797
5798
5799
        compute<<forceNames[atoms[0]]<<".xyz += deltaCross0.xyz;\n";
        compute<<forceNames[atoms[1]]<<".xyz += deltaCross1.xyz;\n";
        compute<<forceNames[atoms[2]]<<".xyz += deltaCross2.xyz;\n";
        compute<<"}\n";
peastman's avatar
peastman committed
5800
        index++;
5801
5802
    }
    index = 0;
peastman's avatar
peastman committed
5803
5804
    for (auto& dihedral : dihedrals) {
        const vector<int>& atoms = dihedral.second;
5805
5806
5807
5808
5809
5810
        string deltaName1 = atomNames[atoms[0]]+atomNames[atoms[1]];
        string deltaName2 = atomNames[atoms[2]]+atomNames[atoms[1]];
        string deltaName3 = atomNames[atoms[2]]+atomNames[atoms[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        compute<<"{\n";
5811
5812
5813
        compute<<"real r = SQRT(delta"<<deltaName2<<".w);\n";
        compute<<"real4 ff;\n";
        compute<<"ff.x = (-dEdDihedral"<<cl.intToString(index)<<"*r)/"<<crossName1<<".w;\n";
5814
5815
        compute<<"ff.y = (delta"<<deltaName1<<".x*delta"<<deltaName2<<".x + delta"<<deltaName1<<".y*delta"<<deltaName2<<".y + delta"<<deltaName1<<".z*delta"<<deltaName2<<".z)/delta"<<deltaName2<<".w;\n";
        compute<<"ff.z = (delta"<<deltaName3<<".x*delta"<<deltaName2<<".x + delta"<<deltaName3<<".y*delta"<<deltaName2<<".y + delta"<<deltaName3<<".z*delta"<<deltaName2<<".z)/delta"<<deltaName2<<".w;\n";
5816
5817
5818
5819
        compute<<"ff.w = (dEdDihedral"<<cl.intToString(index)<<"*r)/"<<crossName2<<".w;\n";
        compute<<"real4 internalF0 = ff.x*"<<crossName1<<";\n";
        compute<<"real4 internalF3 = ff.w*"<<crossName2<<";\n";
        compute<<"real4 s = ff.y*internalF0 - ff.z*internalF3;\n";
5820
5821
5822
5823
5824
        compute<<forceNames[atoms[0]]<<".xyz += internalF0.xyz;\n";
        compute<<forceNames[atoms[1]]<<".xyz += s.xyz-internalF0.xyz;\n";
        compute<<forceNames[atoms[2]]<<".xyz += -s.xyz-internalF3.xyz;\n";
        compute<<forceNames[atoms[3]]<<".xyz += internalF3.xyz;\n";
        compute<<"}\n";
peastman's avatar
peastman committed
5825
        index++;
5826
5827
5828
    }
    cl.getBondedUtilities().addInteraction(atoms, compute.str(), force.getForceGroup());
    map<string, string> replacements;
5829
    replacements["M_PI"] = cl.doubleToString(M_PI);
5830
5831
5832
5833
    cl.getBondedUtilities().addPrefixCode(cl.replaceStrings(OpenCLKernelSources::customCompoundBond, replacements));;
}

double OpenCLCalcCustomCompoundBondForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
peastman's avatar
peastman committed
5834
    if (globals.isInitialized()) {
5835
5836
5837
5838
5839
5840
5841
5842
        bool changed = false;
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
5843
            globals.upload(globalParamValues);
5844
5845
5846
5847
    }
    return 0.0;
}

5848
5849
5850
5851
5852
5853
void OpenCLCalcCustomCompoundBondForceKernel::copyParametersToContext(ContextImpl& context, const CustomCompoundBondForce& force) {
    int numContexts = cl.getPlatformData().contexts.size();
    int startIndex = cl.getContextIndex()*force.getNumBonds()/numContexts;
    int endIndex = (cl.getContextIndex()+1)*force.getNumBonds()/numContexts;
    if (numBonds != endIndex-startIndex)
        throw OpenMMException("updateParametersInContext: The number of bonds has changed");
5854
5855
    if (numBonds == 0)
        return;
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
    
    // Record the per-bond parameters.
    
    vector<vector<cl_float> > paramVector(numBonds);
    vector<int> particles;
    vector<double> parameters;
    for (int i = 0; i < numBonds; i++) {
        force.getBondParameters(startIndex+i, particles, parameters);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (cl_float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
5872
    cl.invalidateMolecules(info);
5873
5874
}

5875
class OpenCLCalcCustomManyParticleForceKernel::ForceInfo : public OpenCLForceInfo {
5876
public:
5877
    ForceInfo(const CustomManyParticleForce& force) : OpenCLForceInfo(0), force(force) {
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        vector<double> params1, params2;
        int type1, type2;
        force.getParticleParameters(particle1, params1, type1);
        force.getParticleParameters(particle2, params2, type2);
        if (type1 != type2)
            return false;
        for (int i = 0; i < (int) params1.size(); i++)
            if (params1[i] != params2[i])
                return false;
        return true;
    }
    int getNumParticleGroups() {
        return force.getNumExclusions();
    }
    void getParticlesInGroup(int index, vector<int>& particles) {
        int particle1, particle2;
        force.getExclusionParticles(index, particle1, particle2);
        particles.resize(2);
        particles[0] = particle1;
        particles[1] = particle2;
    }
    bool areGroupsIdentical(int group1, int group2) {
        return true;
    }
private:
    const CustomManyParticleForce& force;
};

OpenCLCalcCustomManyParticleForceKernel::~OpenCLCalcCustomManyParticleForceKernel() {
    if (params != NULL)
        delete params;
}

void OpenCLCalcCustomManyParticleForceKernel::initialize(const System& system, const CustomManyParticleForce& force) {
    if (!cl.getSupports64BitGlobalAtomics())
        throw OpenMMException("CustomManyParticleForce requires a device that supports 64 bit atomic operations");
    int numParticles = force.getNumParticles();
    int particlesPerSet = force.getNumParticlesPerSet();
    bool centralParticleMode = (force.getPermutationMode() == CustomManyParticleForce::UniqueCentralParticle);
    nonbondedMethod = CalcCustomManyParticleForceKernel::NonbondedMethod(force.getNonbondedMethod());
    forceWorkgroupSize = 128;
    findNeighborsWorkgroupSize = (cl.getSIMDWidth() >= 32 ? 128 : 32);
    
    // Record parameter values.
    
    params = new OpenCLParameterSet(cl, force.getNumPerParticleParameters(), numParticles, "customManyParticleParameters");
    vector<vector<float> > paramVector(numParticles);
    for (int i = 0; i < numParticles; i++) {
        vector<double> parameters;
        int type;
        force.getParticleParameters(i, parameters, type);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (float) parameters[j];
    }
    params->setParameterValues(paramVector);
5936
5937
    info = new ForceInfo(force);
    cl.addForce(info);
5938
5939
5940
5941
5942
5943
5944

    // Record the tabulated functions.

    map<string, Lepton::CustomFunction*> functions;
    vector<pair<string, string> > functionDefinitions;
    vector<const TabulatedFunction*> functionList;
    stringstream tableArgs;
peastman's avatar
peastman committed
5945
    tabulatedFunctions.resize(force.getNumTabulatedFunctions());
5946
5947
5948
5949
5950
5951
5952
5953
    for (int i = 0; i < force.getNumTabulatedFunctions(); i++) {
        functionList.push_back(&force.getTabulatedFunction(i));
        string name = force.getTabulatedFunctionName(i);
        string arrayName = "table"+cl.intToString(i);
        functionDefinitions.push_back(make_pair(name, arrayName));
        functions[name] = cl.getExpressionUtilities().getFunctionPlaceholder(force.getTabulatedFunction(i));
        int width;
        vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(force.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
5954
5955
        tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
        tabulatedFunctions[i].upload(f);
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
        tableArgs << ", __global const float";
        if (width > 1)
            tableArgs << width;
        tableArgs << "* restrict " << arrayName;
    }
    
    // Record information about parameters.

    globalParamNames.resize(force.getNumGlobalParameters());
    globalParamValues.resize(force.getNumGlobalParameters());
    for (int i = 0; i < force.getNumGlobalParameters(); i++) {
        globalParamNames[i] = force.getGlobalParameterName(i);
        globalParamValues[i] = (float) force.getGlobalParameterDefaultValue(i);
    }
    vector<pair<ExpressionTreeNode, string> > variables;
    for (int i = 0; i < particlesPerSet; i++) {
        string index = cl.intToString(i+1);
        variables.push_back(makeVariable("x"+index, "pos"+index+".x"));
        variables.push_back(makeVariable("y"+index, "pos"+index+".y"));
        variables.push_back(makeVariable("z"+index, "pos"+index+".z"));
    }
    for (int i = 0; i < force.getNumPerParticleParameters(); i++) {
        const string& name = force.getPerParticleParameterName(i);
        for (int j = 0; j < particlesPerSet; j++) {
            string index = cl.intToString(j+1);
            variables.push_back(makeVariable(name+index, "params"+params->getParameterSuffix(i, index)));
        }
    }
    if (force.getNumGlobalParameters() > 0) {
peastman's avatar
peastman committed
5985
5986
        globals.initialize<cl_float>(cl, force.getNumGlobalParameters(), "customManyParticleGlobals", CL_MEM_READ_ONLY);
        globals.upload(globalParamValues);
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
        for (int i = 0; i < force.getNumGlobalParameters(); i++) {
            const string& name = force.getGlobalParameterName(i);
            string value = "globals["+cl.intToString(i)+"]";
            variables.push_back(makeVariable(name, value));
        }
    }
    
    // Build data structures for type filters.
    
    vector<int> particleTypesVec;
    vector<int> orderIndexVec;
    vector<std::vector<int> > particleOrderVec;
    int numTypes;
    CustomManyParticleForceImpl::buildFilterArrays(force, numTypes, particleTypesVec, orderIndexVec, particleOrderVec);
    bool hasTypeFilters = (particleOrderVec.size() > 1);
    if (hasTypeFilters) {
peastman's avatar
peastman committed
6003
6004
6005
6006
6007
6008
        particleTypes.initialize<int>(cl, particleTypesVec.size(), "customManyParticleTypes");
        orderIndex.initialize<int>(cl, orderIndexVec.size(), "customManyParticleOrderIndex");
        particleOrder.initialize<int>(cl, particleOrderVec.size()*particlesPerSet, "customManyParticleOrder");
        particleTypes.upload(particleTypesVec);
        orderIndex.upload(orderIndexVec);
        vector<int> flattenedOrder(particleOrder.getSize());
6009
6010
6011
        for (int i = 0; i < (int) particleOrderVec.size(); i++)
            for (int j = 0; j < particlesPerSet; j++)
                flattenedOrder[i*particlesPerSet+j] = particleOrderVec[i][j];
peastman's avatar
peastman committed
6012
        particleOrder.upload(flattenedOrder);
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
    }
    
    // Build data structures for exclusions.
    
    if (force.getNumExclusions() > 0) {
        vector<vector<int> > particleExclusions(numParticles);
        for (int i = 0; i < force.getNumExclusions(); i++) {
            int p1, p2;
            force.getExclusionParticles(i, p1, p2);
            particleExclusions[p1].push_back(p2);
            particleExclusions[p2].push_back(p1);
        }
        vector<int> exclusionsVec;
        vector<int> exclusionStartIndexVec(numParticles+1);
        exclusionStartIndexVec[0] = 0;
        for (int i = 0; i < numParticles; i++) {
            sort(particleExclusions[i].begin(), particleExclusions[i].end());
            exclusionsVec.insert(exclusionsVec.end(), particleExclusions[i].begin(), particleExclusions[i].end());
            exclusionStartIndexVec[i+1] = exclusionsVec.size();
        }
peastman's avatar
peastman committed
6033
6034
6035
6036
        exclusions.initialize<int>(cl, exclusionsVec.size(), "customManyParticleExclusions");
        exclusionStartIndex.initialize<int>(cl, exclusionStartIndexVec.size(), "customManyParticleExclusionStart");
        exclusions.upload(exclusionsVec);
        exclusionStartIndex.upload(exclusionStartIndexVec);
6037
6038
6039
6040
6041
6042
6043
    }
    
    // Build data structures for the neighbor list.
    
    if (nonbondedMethod != NoCutoff) {
        int numAtomBlocks = cl.getNumAtomBlocks();
        int elementSize = (cl.getUseDoublePrecision() ? sizeof(double) : sizeof(float));
peastman's avatar
peastman committed
6044
6045
6046
6047
6048
        blockCenter.initialize(cl, numAtomBlocks, 4*elementSize, "blockCenter");
        blockBoundingBox.initialize(cl, numAtomBlocks, 4*elementSize, "blockBoundingBox");
        numNeighborPairs.initialize<int>(cl, 1, "customManyParticleNumNeighborPairs");
        neighborStartIndex.initialize<int>(cl, numParticles+1, "customManyParticleNeighborStartIndex");
        numNeighborsForAtom.initialize<int>(cl, numParticles, "customManyParticleNumNeighborsForAtom");
6049
6050
6051
6052
6053

        // Select a size for the array that holds the neighbor list.  We have to make a fairly
        // arbitrary guess, but if this turns out to be too small we'll increase it later.

        maxNeighborPairs = 150*numParticles;
peastman's avatar
peastman committed
6054
6055
        neighborPairs.initialize<mm_int2>(cl, maxNeighborPairs, "customManyParticleNeighborPairs");
        neighbors.initialize<int>(cl, maxNeighborPairs, "customManyParticleNeighbors");
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
    }

    // Now to generate the kernel.  First, it needs to calculate all distances, angles,
    // and dihedrals the expression depends on.

    map<string, vector<int> > distances;
    map<string, vector<int> > angles;
    map<string, vector<int> > dihedrals;
    Lepton::ParsedExpression energyExpression = CustomManyParticleForceImpl::prepareExpression(force, functions, distances, angles, dihedrals);
    map<string, Lepton::ParsedExpression> forceExpressions;
    set<string> computedDeltas;
    vector<string> atomNames, posNames;
    for (int i = 0; i < particlesPerSet; i++) {
        string index = cl.intToString(i+1);
        atomNames.push_back("P"+index);
        posNames.push_back("pos"+index);
    }
    stringstream compute;
    int index = 0;
peastman's avatar
peastman committed
6075
6076
    for (auto& distance : distances) {
        const vector<int>& atoms = distance.second;
6077
6078
        string deltaName = atomNames[atoms[0]]+atomNames[atoms[1]];
        if (computedDeltas.count(deltaName) == 0) {
6079
            compute<<"real4 delta"<<deltaName<<" = delta("<<posNames[atoms[0]]<<", "<<posNames[atoms[1]]<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
6080
6081
6082
            computedDeltas.insert(deltaName);
        }
        compute<<"real r_"<<deltaName<<" = sqrt(delta"<<deltaName<<".w);\n";
peastman's avatar
peastman committed
6083
6084
6085
        variables.push_back(makeVariable(distance.first, "r_"+deltaName));
        forceExpressions["real dEdDistance"+cl.intToString(index)+" = "] = energyExpression.differentiate(distance.first).optimize();
        index++;
6086
6087
    }
    index = 0;
peastman's avatar
peastman committed
6088
6089
    for (auto& angle : angles) {
        const vector<int>& atoms = angle.second;
6090
6091
6092
6093
        string deltaName1 = atomNames[atoms[1]]+atomNames[atoms[0]];
        string deltaName2 = atomNames[atoms[1]]+atomNames[atoms[2]];
        string angleName = "angle_"+atomNames[atoms[0]]+atomNames[atoms[1]]+atomNames[atoms[2]];
        if (computedDeltas.count(deltaName1) == 0) {
6094
            compute<<"real4 delta"<<deltaName1<<" = delta("<<posNames[atoms[1]]<<", "<<posNames[atoms[0]]<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
6095
6096
6097
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
6098
            compute<<"real4 delta"<<deltaName2<<" = delta("<<posNames[atoms[1]]<<", "<<posNames[atoms[2]]<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
6099
6100
6101
            computedDeltas.insert(deltaName2);
        }
        compute<<"real "<<angleName<<" = computeAngle(delta"<<deltaName1<<", delta"<<deltaName2<<");\n";
peastman's avatar
peastman committed
6102
6103
6104
        variables.push_back(makeVariable(angle.first, angleName));
        forceExpressions["real dEdAngle"+cl.intToString(index)+" = "] = energyExpression.differentiate(angle.first).optimize();
        index++;
6105
6106
    }
    index = 0;
peastman's avatar
peastman committed
6107
6108
    for (auto& dihedral : dihedrals) {
        const vector<int>& atoms = dihedral.second;
6109
6110
6111
6112
6113
6114
6115
        string deltaName1 = atomNames[atoms[0]]+atomNames[atoms[1]];
        string deltaName2 = atomNames[atoms[2]]+atomNames[atoms[1]];
        string deltaName3 = atomNames[atoms[2]]+atomNames[atoms[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        string dihedralName = "dihedral_"+atomNames[atoms[0]]+atomNames[atoms[1]]+atomNames[atoms[2]]+atomNames[atoms[3]];
        if (computedDeltas.count(deltaName1) == 0) {
6116
            compute<<"real4 delta"<<deltaName1<<" = delta("<<posNames[atoms[0]]<<", "<<posNames[atoms[1]]<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
6117
6118
6119
            computedDeltas.insert(deltaName1);
        }
        if (computedDeltas.count(deltaName2) == 0) {
6120
            compute<<"real4 delta"<<deltaName2<<" = delta("<<posNames[atoms[2]]<<", "<<posNames[atoms[1]]<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
6121
6122
6123
            computedDeltas.insert(deltaName2);
        }
        if (computedDeltas.count(deltaName3) == 0) {
6124
            compute<<"real4 delta"<<deltaName3<<" = delta("<<posNames[atoms[2]]<<", "<<posNames[atoms[3]]<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ);\n";
6125
6126
6127
6128
6129
6130
            computedDeltas.insert(deltaName3);
        }
        compute<<"real4 "<<crossName1<<" = computeCross(delta"<<deltaName1<<", delta"<<deltaName2<<");\n";
        compute<<"real4 "<<crossName2<<" = computeCross(delta"<<deltaName2<<", delta"<<deltaName3<<");\n";
        compute<<"real "<<dihedralName<<" = computeAngle("<<crossName1<<", "<<crossName2<<");\n";
        compute<<dihedralName<<" *= (delta"<<deltaName1<<".x*"<<crossName2<<".x + delta"<<deltaName1<<".y*"<<crossName2<<".y + delta"<<deltaName1<<".z*"<<crossName2<<".z < 0 ? -1 : 1);\n";
peastman's avatar
peastman committed
6131
6132
6133
        variables.push_back(makeVariable(dihedral.first, dihedralName));
        forceExpressions["real dEdDihedral"+cl.intToString(index)+" = "] = energyExpression.differentiate(dihedral.first).optimize();
        index++;
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
    }

    // Now evaluate the expressions.

    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
        compute<<buffer.getType()<<" params"<<(i+1)<<" = global_params"<<(i+1)<<"[index];\n";
    }
    forceExpressions["energy += "] = energyExpression;
    compute << cl.getExpressionUtilities().createExpressions(forceExpressions, variables, functionList, functionDefinitions, "temp");

    // Apply forces to atoms.

    vector<string> forceNames;
    for (int i = 0; i < particlesPerSet; i++) {
        string istr = cl.intToString(i+1);
        string forceName = "force"+istr;
        forceNames.push_back(forceName);
        compute<<"real4 "<<forceName<<" = (real4) 0;\n";
        compute<<"{\n";
        Lepton::ParsedExpression forceExpressionX = energyExpression.differentiate("x"+istr).optimize();
        Lepton::ParsedExpression forceExpressionY = energyExpression.differentiate("y"+istr).optimize();
        Lepton::ParsedExpression forceExpressionZ = energyExpression.differentiate("z"+istr).optimize();
        map<string, Lepton::ParsedExpression> expressions;
        if (!isZeroExpression(forceExpressionX))
            expressions[forceName+".x -= "] = forceExpressionX;
        if (!isZeroExpression(forceExpressionY))
            expressions[forceName+".y -= "] = forceExpressionY;
        if (!isZeroExpression(forceExpressionZ))
            expressions[forceName+".z -= "] = forceExpressionZ;
        if (expressions.size() > 0)
            compute<<cl.getExpressionUtilities().createExpressions(expressions, variables, functionList, functionDefinitions, "coordtemp");
        compute<<"}\n";
    }
    index = 0;
peastman's avatar
peastman committed
6169
6170
    for (auto& distance : distances) {
        const vector<int>& atoms = distance.second;
6171
6172
6173
6174
        string deltaName = atomNames[atoms[0]]+atomNames[atoms[1]];
        string value = "(dEdDistance"+cl.intToString(index)+"/r_"+deltaName+")*delta"+deltaName+".xyz";
        compute<<forceNames[atoms[0]]<<".xyz += "<<"-"<<value<<";\n";
        compute<<forceNames[atoms[1]]<<".xyz += "<<value<<";\n";
peastman's avatar
peastman committed
6175
        index++;
6176
6177
    }
    index = 0;
peastman's avatar
peastman committed
6178
6179
    for (auto& angle : angles) {
        const vector<int>& atoms = angle.second;
6180
6181
6182
6183
        string deltaName1 = atomNames[atoms[1]]+atomNames[atoms[0]];
        string deltaName2 = atomNames[atoms[1]]+atomNames[atoms[2]];
        compute<<"{\n";
        compute<<"real4 crossProd = cross(delta"<<deltaName2<<", delta"<<deltaName1<<");\n";
6184
        compute<<"real lengthCross = max(SQRT(dot(crossProd, crossProd)), (real) 1e-6f);\n";
6185
6186
6187
6188
6189
6190
6191
        compute<<"real4 deltaCross0 = -cross(delta"<<deltaName1<<", crossProd)*dEdAngle"<<cl.intToString(index)<<"/(delta"<<deltaName1<<".w*lengthCross);\n";
        compute<<"real4 deltaCross2 = cross(delta"<<deltaName2<<", crossProd)*dEdAngle"<<cl.intToString(index)<<"/(delta"<<deltaName2<<".w*lengthCross);\n";
        compute<<"real4 deltaCross1 = -(deltaCross0+deltaCross2);\n";
        compute<<forceNames[atoms[0]]<<".xyz += deltaCross0.xyz;\n";
        compute<<forceNames[atoms[1]]<<".xyz += deltaCross1.xyz;\n";
        compute<<forceNames[atoms[2]]<<".xyz += deltaCross2.xyz;\n";
        compute<<"}\n";
peastman's avatar
peastman committed
6192
        index++;
6193
6194
    }
    index = 0;
peastman's avatar
peastman committed
6195
6196
    for (auto& dihedral : dihedrals) {
        const vector<int>& atoms = dihedral.second;
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
        string deltaName1 = atomNames[atoms[0]]+atomNames[atoms[1]];
        string deltaName2 = atomNames[atoms[2]]+atomNames[atoms[1]];
        string deltaName3 = atomNames[atoms[2]]+atomNames[atoms[3]];
        string crossName1 = "cross_"+deltaName1+"_"+deltaName2;
        string crossName2 = "cross_"+deltaName2+"_"+deltaName3;
        compute<<"{\n";
        compute<<"real r = sqrt(delta"<<deltaName2<<".w);\n";
        compute<<"real4 ff;\n";
        compute<<"ff.x = (-dEdDihedral"<<cl.intToString(index)<<"*r)/"<<crossName1<<".w;\n";
        compute<<"ff.y = (delta"<<deltaName1<<".x*delta"<<deltaName2<<".x + delta"<<deltaName1<<".y*delta"<<deltaName2<<".y + delta"<<deltaName1<<".z*delta"<<deltaName2<<".z)/delta"<<deltaName2<<".w;\n";
        compute<<"ff.z = (delta"<<deltaName3<<".x*delta"<<deltaName2<<".x + delta"<<deltaName3<<".y*delta"<<deltaName2<<".y + delta"<<deltaName3<<".z*delta"<<deltaName2<<".z)/delta"<<deltaName2<<".w;\n";
        compute<<"ff.w = (dEdDihedral"<<cl.intToString(index)<<"*r)/"<<crossName2<<".w;\n";
        compute<<"real4 internalF0 = ff.x*"<<crossName1<<";\n";
        compute<<"real4 internalF3 = ff.w*"<<crossName2<<";\n";
        compute<<"real4 s = ff.y*internalF0 - ff.z*internalF3;\n";
        compute<<forceNames[atoms[0]]<<".xyz += internalF0.xyz;\n";
        compute<<forceNames[atoms[1]]<<".xyz += s.xyz-internalF0.xyz;\n";
        compute<<forceNames[atoms[2]]<<".xyz += -s.xyz-internalF3.xyz;\n";
        compute<<forceNames[atoms[3]]<<".xyz += internalF3.xyz;\n";
        compute<<"}\n";
peastman's avatar
peastman committed
6217
        index++;
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
    }
    
    // Store forces to global memory.
    
    for (int i = 0; i < particlesPerSet; i++)
        compute<<"storeForce(atom"<<(i+1)<<", "<<forceNames[i]<<", forceBuffers);\n";
    
    // Create other replacements that depend on the number of particles per set.
    
    stringstream numCombinations, atomsForCombination, isValidCombination, permute, loadData, verifyCutoff, verifyExclusions;
    if (hasTypeFilters) {
        permute<<"int particleSet[] = {";
        for (int i = 0; i < particlesPerSet; i++) {
            permute<<"p"<<(i+1);
            if (i < particlesPerSet-1)
                permute<<", ";
        }
        permute<<"};\n";
    }
    for (int i = 0; i < particlesPerSet; i++) {
        if (hasTypeFilters)
peastman's avatar
Bug fix  
peastman committed
6239
            permute<<"int atom"<<(i+1)<<" = particleSet[particleOrder["<<particlesPerSet<<"*order+"<<i<<"]];\n";
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
        else
            permute<<"int atom"<<(i+1)<<" = p"<<(i+1)<<";\n";
        loadData<<"real4 pos"<<(i+1)<<" = posq[atom"<<(i+1)<<"];\n";
        for (int j = 0; j < (int) params->getBuffers().size(); j++)
            loadData<<params->getBuffers()[j].getType()<<" params"<<(j+1)<<(i+1)<<" = global_params"<<(j+1)<<"[atom"<<(i+1)<<"];\n";
    }
    if (centralParticleMode) {
        for (int i = 1; i < particlesPerSet; i++) {
            if (i > 1)
                isValidCombination<<" && p"<<(i+1)<<">p"<<i<<" && ";
            isValidCombination<<"p"<<(i+1)<<"!=p1";
        }
    }
    else {
        for (int i = 2; i < particlesPerSet; i++) {
            if (i > 2)
                isValidCombination<<" && ";
            isValidCombination<<"a"<<(i+1)<<">a"<<i;
        }
    }
    atomsForCombination<<"int tempIndex = index;\n";
    for (int i = 1; i < particlesPerSet; i++) {
        if (i > 1)
            numCombinations<<"*";
        numCombinations<<"numNeighbors";
        if (centralParticleMode)
            atomsForCombination<<"int a"<<(i+1)<<" = tempIndex%numNeighbors;\n";
        else
            atomsForCombination<<"int a"<<(i+1)<<" = 1+tempIndex%numNeighbors;\n";
        if (i < particlesPerSet-1)
            atomsForCombination<<"tempIndex /= numNeighbors;\n";
    }
    if (particlesPerSet > 2) {
        if (centralParticleMode)
            atomsForCombination<<"a2 = (a3%2 == 0 ? a2 : numNeighbors-a2-1);\n";
        else
            atomsForCombination<<"a2 = (a3%2 == 0 ? a2 : numNeighbors-a2+1);\n";
    }
    for (int i = 1; i < particlesPerSet; i++) {
        if (nonbondedMethod == NoCutoff) {
            if (centralParticleMode)
                atomsForCombination<<"int p"<<(i+1)<<" = a"<<(i+1)<<";\n";
            else
                atomsForCombination<<"int p"<<(i+1)<<" = p1+a"<<(i+1)<<";\n";
        }
        else {
            if (centralParticleMode)
                atomsForCombination<<"int p"<<(i+1)<<" = neighbors[firstNeighbor+a"<<(i+1)<<"];\n";
            else
                atomsForCombination<<"int p"<<(i+1)<<" = neighbors[firstNeighbor-1+a"<<(i+1)<<"];\n";
        }
    }
    if (nonbondedMethod != NoCutoff) {
        for (int i = 1; i < particlesPerSet; i++)
            verifyCutoff<<"real4 pos"<<(i+1)<<" = posq[p"<<(i+1)<<"];\n";
        if (!centralParticleMode) {
            for (int i = 1; i < particlesPerSet; i++) {
                for (int j = i+1; j < particlesPerSet; j++)
6298
                    verifyCutoff<<"includeInteraction &= (delta(pos"<<(i+1)<<", pos"<<(j+1)<<", periodicBoxSize, invPeriodicBoxSize, periodicBoxVecX, periodicBoxVecY, periodicBoxVecZ).w < CUTOFF_SQUARED);\n";
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
            }
        }
    }
    if (force.getNumExclusions() > 0) {
        int startCheckFrom = (nonbondedMethod == NoCutoff ? 0 : 1);
        for (int i = startCheckFrom; i < particlesPerSet; i++)
            for (int j = i+1; j < particlesPerSet; j++)
                verifyExclusions<<"includeInteraction &= !isInteractionExcluded(p"<<(i+1)<<", p"<<(j+1)<<", exclusions, exclusionStartIndex);\n";
    }
    string computeTypeIndex = "particleTypes[p"+cl.intToString(particlesPerSet)+"]";
    for (int i = particlesPerSet-2; i >= 0; i--)
        computeTypeIndex = "particleTypes[p"+cl.intToString(i+1)+"]+"+cl.intToString(numTypes)+"*("+computeTypeIndex+")";
    
    // Create replacements for extra arguments.
    
    stringstream extraArgs;
    if (force.getNumGlobalParameters() > 0)
        extraArgs << ", __global const float* globals";
    for (int i = 0; i < (int) params->getBuffers().size(); i++) {
        OpenCLNonbondedUtilities::ParameterInfo& buffer = params->getBuffers()[i];
        extraArgs<<", __global const "<<buffer.getType()<<"* restrict global_params"<<(i+1);
    }

    // Create the kernels.

    map<string, string> replacements;
    replacements["COMPUTE_INTERACTION"] = compute.str();
    replacements["NUM_CANDIDATE_COMBINATIONS"] = numCombinations.str();
    replacements["FIND_ATOMS_FOR_COMBINATION_INDEX"] = atomsForCombination.str();
    replacements["IS_VALID_COMBINATION"] = isValidCombination.str();
    replacements["VERIFY_CUTOFF"] = verifyCutoff.str();
    replacements["VERIFY_EXCLUSIONS"] = verifyExclusions.str();
    replacements["PERMUTE_ATOMS"] = permute.str();
    replacements["LOAD_PARTICLE_DATA"] = loadData.str();
    replacements["COMPUTE_TYPE_INDEX"] = computeTypeIndex;
    replacements["PARAMETER_ARGUMENTS"] = extraArgs.str()+tableArgs.str();
    map<string, string> defines;
    if (nonbondedMethod != NoCutoff)
        defines["USE_CUTOFF"] = "1";
    if (nonbondedMethod == CutoffPeriodic)
        defines["USE_PERIODIC"] = "1";
    if (centralParticleMode)
        defines["USE_CENTRAL_PARTICLE"] = "1";
    if (hasTypeFilters)
        defines["USE_FILTERS"] = "1";
    if (force.getNumExclusions() > 0)
        defines["USE_EXCLUSIONS"] = "1";
    defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
    defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
    defines["M_PI"] = cl.doubleToString(M_PI);
    defines["CUTOFF_SQUARED"] = cl.doubleToString(force.getCutoffDistance()*force.getCutoffDistance());
    defines["TILE_SIZE"] = cl.intToString(OpenCLContext::TileSize);
    defines["NUM_BLOCKS"] = cl.intToString(cl.getNumAtomBlocks());
    defines["FIND_NEIGHBORS_WORKGROUP_SIZE"] = cl.intToString(findNeighborsWorkgroupSize);
    cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customManyParticle, replacements), defines);
    forceKernel = cl::Kernel(program, "computeInteraction");
    blockBoundsKernel = cl::Kernel(program, "findBlockBounds");
    neighborsKernel = cl::Kernel(program, "findNeighbors");
    startIndicesKernel = cl::Kernel(program, "computeNeighborStartIndices");
    copyPairsKernel = cl::Kernel(program, "copyPairsToNeighborList");
}

double OpenCLCalcCustomManyParticleForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
    if (!hasInitializedKernel) {
        hasInitializedKernel = true;
        
        // Set arguments for the force kernel.
        
        int index = 0;
6368
        forceKernel.setArg<cl::Buffer>(index++, cl.getLongForceBuffer().getDeviceBuffer());
6369
6370
        forceKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
6371
6372
        setPeriodicBoxArgs(cl, forceKernel, index);
        index += 5;
6373
        if (nonbondedMethod != NoCutoff) {
peastman's avatar
peastman committed
6374
6375
            forceKernel.setArg<cl::Buffer>(index++, neighbors.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(index++, neighborStartIndex.getDeviceBuffer());
6376
        }
peastman's avatar
peastman committed
6377
6378
6379
6380
        if (particleTypes.isInitialized()) {
            forceKernel.setArg<cl::Buffer>(index++, particleTypes.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(index++, orderIndex.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(index++, particleOrder.getDeviceBuffer());
6381
        }
peastman's avatar
peastman committed
6382
6383
6384
        if (exclusions.isInitialized()) {
            forceKernel.setArg<cl::Buffer>(index++, exclusions.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(index++, exclusionStartIndex.getDeviceBuffer());
6385
        }
peastman's avatar
peastman committed
6386
6387
        if (globals.isInitialized())
            forceKernel.setArg<cl::Buffer>(index++, globals.getDeviceBuffer());
peastman's avatar
peastman committed
6388
        for (auto& buffer : params->getBuffers())
6389
            forceKernel.setArg<cl::Memory>(index++, buffer.getMemory());
peastman's avatar
peastman committed
6390
6391
        for (auto& function : tabulatedFunctions)
            forceKernel.setArg<cl::Buffer>(index++, function.getDeviceBuffer());
6392
6393
6394
6395
6396
        
        if (nonbondedMethod != NoCutoff) {
            // Set arguments for the block bounds kernel.

            index = 0;
6397
6398
            setPeriodicBoxArgs(cl, blockBoundsKernel, index);
            index += 5;
6399
            blockBoundsKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
6400
6401
6402
            blockBoundsKernel.setArg<cl::Buffer>(index++, blockCenter.getDeviceBuffer());
            blockBoundsKernel.setArg<cl::Buffer>(index++, blockBoundingBox.getDeviceBuffer());
            blockBoundsKernel.setArg<cl::Buffer>(index++, numNeighborPairs.getDeviceBuffer());
6403
6404
6405
6406

            // Set arguments for the neighbor list kernel.

            index = 0;
6407
6408
            setPeriodicBoxArgs(cl, neighborsKernel, index);
            index += 5;
6409
            neighborsKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
6410
6411
6412
6413
6414
            neighborsKernel.setArg<cl::Buffer>(index++, blockCenter.getDeviceBuffer());
            neighborsKernel.setArg<cl::Buffer>(index++, blockBoundingBox.getDeviceBuffer());
            neighborsKernel.setArg<cl::Buffer>(index++, neighborPairs.getDeviceBuffer());
            neighborsKernel.setArg<cl::Buffer>(index++, numNeighborPairs.getDeviceBuffer());
            neighborsKernel.setArg<cl::Buffer>(index++, numNeighborsForAtom.getDeviceBuffer());
6415
            index++;
peastman's avatar
peastman committed
6416
6417
6418
            if (exclusions.isInitialized()) {
                neighborsKernel.setArg<cl::Buffer>(index++, exclusions.getDeviceBuffer());
                neighborsKernel.setArg<cl::Buffer>(index++, exclusionStartIndex.getDeviceBuffer());
6419
6420
6421
6422
6423
            }
            
            // Set arguments for the kernel to find neighbor list start indices.
            
            index = 0;
peastman's avatar
peastman committed
6424
6425
6426
            startIndicesKernel.setArg<cl::Buffer>(index++, numNeighborsForAtom.getDeviceBuffer());
            startIndicesKernel.setArg<cl::Buffer>(index++, neighborStartIndex.getDeviceBuffer());
            startIndicesKernel.setArg<cl::Buffer>(index++, numNeighborPairs.getDeviceBuffer());
6427
6428
6429
6430

            // Set arguments for the kernel to assemble the final neighbor list.
            
            index = 0;
peastman's avatar
peastman committed
6431
6432
6433
            copyPairsKernel.setArg<cl::Buffer>(index++, neighborPairs.getDeviceBuffer());
            copyPairsKernel.setArg<cl::Buffer>(index++, neighbors.getDeviceBuffer());
            copyPairsKernel.setArg<cl::Buffer>(index++, numNeighborPairs.getDeviceBuffer());
6434
            index++;
peastman's avatar
peastman committed
6435
6436
            copyPairsKernel.setArg<cl::Buffer>(index++, numNeighborsForAtom.getDeviceBuffer());
            copyPairsKernel.setArg<cl::Buffer>(index++, neighborStartIndex.getDeviceBuffer());
6437
6438
       }
    }
peastman's avatar
peastman committed
6439
    if (globals.isInitialized()) {
6440
6441
6442
6443
6444
6445
6446
6447
        bool changed = false;
        for (int i = 0; i < (int) globalParamNames.size(); i++) {
            cl_float value = (cl_float) context.getParameter(globalParamNames[i]);
            if (value != globalParamValues[i])
                changed = true;
            globalParamValues[i] = value;
        }
        if (changed)
peastman's avatar
peastman committed
6448
            globals.upload(globalParamValues);
6449
6450
6451
6452
6453
    }
    while (true) {
        int* numPairs = (int*) cl.getPinnedBuffer();
        cl::Event event;
        if (nonbondedMethod != NoCutoff) {
6454
            neighborsKernel.setArg<int>(11, maxNeighborPairs);
6455
6456
6457
6458
6459
6460
6461
6462
            startIndicesKernel.setArg<int>(3, maxNeighborPairs);
            copyPairsKernel.setArg<int>(3, maxNeighborPairs);
            cl.executeKernel(blockBoundsKernel, cl.getNumAtomBlocks());
            cl.executeKernel(neighborsKernel, cl.getNumAtoms(), findNeighborsWorkgroupSize);

            // We need to make sure there was enough memory for the neighbor list.  Download the
            // information asynchronously so kernels can be running at the same time.

peastman's avatar
peastman committed
6463
            numNeighborPairs.download(numPairs, false);
6464
6465
6466
6467
            cl.getQueue().enqueueMarker(&event);
            cl.executeKernel(startIndicesKernel, 256, 256);
            cl.executeKernel(copyPairsKernel, maxNeighborPairs);
        }
6468
6469
        int maxThreads = min(cl.getNumAtoms()*forceWorkgroupSize, cl.getEnergyBuffer().getSize());
        cl.executeKernel(forceKernel, maxThreads, forceWorkgroupSize);
6470
6471
6472
6473
6474
6475
6476
6477
        if (nonbondedMethod != NoCutoff) {
            // Make sure there was enough memory for the neighbor list.

            event.wait();
            if (*numPairs > maxNeighborPairs) {
                // Resize the arrays and run the calculation again.

                maxNeighborPairs = (int) (1.1*(*numPairs));
peastman's avatar
peastman committed
6478
6479
6480
6481
6482
6483
                neighborPairs.resize(maxNeighborPairs);
                neighbors.resize(maxNeighborPairs);
                forceKernel.setArg<cl::Buffer>(8, neighbors.getDeviceBuffer());
                neighborsKernel.setArg<cl::Buffer>(8, neighborPairs.getDeviceBuffer());
                copyPairsKernel.setArg<cl::Buffer>(0, neighborPairs.getDeviceBuffer());
                copyPairsKernel.setArg<cl::Buffer>(1, neighbors.getDeviceBuffer());
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
                continue;
            }
        }
        break;
    }
    return 0.0;
}

void OpenCLCalcCustomManyParticleForceKernel::copyParametersToContext(ContextImpl& context, const CustomManyParticleForce& force) {
    int numParticles = force.getNumParticles();
    if (numParticles != cl.getNumAtoms())
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
    
    // Record the per-particle parameters.
    
    vector<vector<float> > paramVector(numParticles);
    vector<double> parameters;
    int type;
    for (int i = 0; i < numParticles; i++) {
        force.getParticleParameters(i, parameters, type);
        paramVector[i].resize(parameters.size());
        for (int j = 0; j < (int) parameters.size(); j++)
            paramVector[i][j] = (float) parameters[j];
    }
    params->setParameterValues(paramVector);
    
    // Mark that the current reordering may be invalid.
    
6512
    cl.invalidateMolecules(info);
6513
6514
}

6515
class OpenCLCalcGayBerneForceKernel::ForceInfo : public OpenCLForceInfo {
6516
public:
6517
    ForceInfo(int requiredBuffers, const GayBerneForce& force) : OpenCLForceInfo(requiredBuffers), force(force) {
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        int xparticle1, yparticle1;
        double sigma1, epsilon1, sx1, sy1, sz1, ex1, ey1, ez1;
        int xparticle2, yparticle2;
        double sigma2, epsilon2, sx2, sy2, sz2, ex2, ey2, ez2;
        force.getParticleParameters(particle1, sigma1, epsilon1, xparticle1, yparticle1, sx1, sy1, sz1, ex1, ey1, ez1);
        force.getParticleParameters(particle2, sigma2, epsilon2, xparticle2, yparticle2, sx2, sy2, sz2, ex2, ey2, ez2);
        return (sigma1 == sigma2 && epsilon1 == epsilon2 && sx1 == sx2 && sy1 == sy2 && sz1 == sz2 && ex1 == ex2 && ey1 == ey2 && ez1 == ez2);
    }
    int getNumParticleGroups() {
        return force.getNumExceptions()+force.getNumParticles();
    }
    void getParticlesInGroup(int index, vector<int>& particles) {
        if (index < force.getNumExceptions()) {
            int particle1, particle2;
            double sigma, epsilon;
            force.getExceptionParameters(index, particle1, particle2, sigma, epsilon);
            particles.resize(2);
            particles[0] = particle1;
            particles[1] = particle2;
        }
        else {
            int particle = index-force.getNumExceptions();
            int xparticle, yparticle;
            double sigma, epsilon, sx, sy, sz, ex, ey, ez;
            force.getParticleParameters(particle, sigma, epsilon, xparticle, yparticle, sx, sy, sz, ex, ey, ez);
            particles.clear();
            particles.push_back(particle);
            if (xparticle > -1)
                particles.push_back(xparticle);
            if (yparticle > -1)
                particles.push_back(yparticle);
        }
    }
    bool areGroupsIdentical(int group1, int group2) {
        if (group1 < force.getNumExceptions() && group2 < force.getNumExceptions()) {
            int particle1, particle2;
            double sigma1, sigma2, epsilon1, epsilon2;
            force.getExceptionParameters(group1, particle1, particle2, sigma1, epsilon1);
            force.getExceptionParameters(group2, particle1, particle2, sigma2, epsilon2);
            return (sigma1 == sigma2 && epsilon1 == epsilon2);
        }
        return true;
    }
private:
    const GayBerneForce& force;
};

class OpenCLCalcGayBerneForceKernel::ReorderListener : public OpenCLContext::ReorderListener {
public:
    ReorderListener(OpenCLCalcGayBerneForceKernel& owner) : owner(owner) {
    }
    void execute() {
        owner.sortAtoms();
    }
private:
    OpenCLCalcGayBerneForceKernel& owner;
};

void OpenCLCalcGayBerneForceKernel::initialize(const System& system, const GayBerneForce& force) {
6579
6580
    if (!cl.getSupports64BitGlobalAtomics())
        throw OpenMMException("GayBerneForce requires a device that supports 64 bit atomic operations");
6581
6582
6583
6584

    // Initialize interactions.

    int numParticles = force.getNumParticles();
peastman's avatar
peastman committed
6585
6586
6587
6588
6589
6590
6591
6592
    sigParams.initialize<mm_float4>(cl, cl.getPaddedNumAtoms(), "sigParams");
    epsParams.initialize<mm_float2>(cl, cl.getPaddedNumAtoms(), "epsParams");
    scale.initialize<mm_float4>(cl, cl.getPaddedNumAtoms(), "scale");
    axisParticleIndices.initialize<mm_int2>(cl, cl.getPaddedNumAtoms(), "axisParticleIndices");
    sortedParticles.initialize<cl_int>(cl, cl.getPaddedNumAtoms(), "sortedParticles");
    aMatrix.initialize<cl_float>(cl, 9*cl.getPaddedNumAtoms(), "aMatrix");
    bMatrix.initialize<cl_float>(cl, 9*cl.getPaddedNumAtoms(), "bMatrix");
    gMatrix.initialize<cl_float>(cl, 9*cl.getPaddedNumAtoms(), "gMatrix");
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
    vector<mm_float4> sigParamsVector(cl.getPaddedNumAtoms(), mm_float4(0, 0, 0, 0));
    vector<mm_float2> epsParamsVector(cl.getPaddedNumAtoms(), mm_float2(0, 0));
    vector<mm_float4> scaleVector(cl.getPaddedNumAtoms(), mm_float4(0, 0, 0, 0));
    vector<mm_int2> axisParticleVector(cl.getPaddedNumAtoms(), mm_int2(0, 0));
    isRealParticle.resize(cl.getPaddedNumAtoms());
    for (int i = 0; i < numParticles; i++) {
        int xparticle, yparticle;
        double sigma, epsilon, sx, sy, sz, ex, ey, ez;
        force.getParticleParameters(i, sigma, epsilon, xparticle, yparticle, sx, sy, sz, ex, ey, ez);
        axisParticleVector[i] = mm_int2(xparticle, yparticle);
        sigParamsVector[i] = mm_float4((float) (0.5*sigma), (float) (0.25*sx*sx), (float) (0.25*sy*sy), (float) (0.25*sz*sz));
        epsParamsVector[i] = mm_float2((float) sqrt(epsilon), (float) (0.125*(sx*sy + sz*sz)*sqrt(sx*sy)));
        scaleVector[i] = mm_float4((float) (1/sqrt(ex)), (float) (1/sqrt(ey)), (float) (1/sqrt(ez)), 0);
6606
        isRealParticle[i] = (epsilon != 0.0);
6607
    }
peastman's avatar
peastman committed
6608
6609
6610
6611
    sigParams.upload(sigParamsVector);
    epsParams.upload(epsParamsVector);
    scale.upload(scaleVector);
    axisParticleIndices.upload(axisParticleVector);
6612
    
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
    // Record exceptions and exclusions.

    vector<mm_float2> exceptionParamsVec;
    for (int i = 0; i < force.getNumExceptions(); i++) {
        int particle1, particle2;
        double sigma, epsilon;
        force.getExceptionParameters(i, particle1, particle2, sigma, epsilon);
        if (epsilon != 0.0) {
            exceptionParamsVec.push_back(mm_float2((float) sigma, (float) epsilon));
            exceptionAtoms.push_back(make_pair(particle1, particle2));
            isRealParticle[particle1] = true;
            isRealParticle[particle2] = true;
        }
        if (isRealParticle[particle1] && isRealParticle[particle2])
            excludedPairs.push_back(pair<int, int>(particle1, particle2));
    }
    numRealParticles = 0;
    for (int i = 0; i < isRealParticle.size(); i++)
        if (isRealParticle[i])
            numRealParticles++;
    int numExceptions = exceptionParamsVec.size();
peastman's avatar
peastman committed
6634
6635
6636
6637
    exclusions.initialize<cl_int>(cl, max(1, (int) excludedPairs.size()), "exclusions");
    exclusionStartIndex.initialize<cl_int>(cl, numRealParticles+1, "exclusionStartIndex");
    exceptionParticles.initialize<mm_int4>(cl, max(1, numExceptions), "exceptionParticles");
    exceptionParams.initialize<mm_float2>(cl, max(1, numExceptions), "exceptionParams");
6638
    if (numExceptions > 0)
peastman's avatar
peastman committed
6639
        exceptionParams.upload(exceptionParamsVec);
6640
    
6641
6642
6643
6644
    // Create data structures used for the neighbor list.

    int numAtomBlocks = (numRealParticles+31)/32;
    int elementSize = (cl.getUseDoublePrecision() ? sizeof(cl_double) : sizeof(cl_float));
peastman's avatar
peastman committed
6645
6646
6647
    blockCenter.initialize(cl, numAtomBlocks, 4*elementSize, "blockCenter");
    blockBoundingBox.initialize(cl, numAtomBlocks, 4*elementSize, "blockBoundingBox");
    sortedPos.initialize(cl, numRealParticles, 4*elementSize, "sortedPos");
6648
    maxNeighborBlocks = numRealParticles*2;
peastman's avatar
peastman committed
6649
6650
6651
    neighbors.initialize<cl_int>(cl, maxNeighborBlocks*32, "neighbors");
    neighborIndex.initialize<cl_int>(cl, maxNeighborBlocks, "neighborIndex");
    neighborBlockCount.initialize<cl_int>(cl, 1, "neighborBlockCount");
6652

6653
    // Create array for accumulating torques.
6654
    
peastman's avatar
peastman committed
6655
6656
    torque.initialize<cl_long>(cl, 3*cl.getPaddedNumAtoms(), "torque");
    cl.addAutoclearBuffer(torque);
6657
6658
6659
6660
6661
6662
6663
6664

    // Create the kernels.
    
    nonbondedMethod = force.getNonbondedMethod();
    bool useCutoff = (nonbondedMethod != GayBerneForce::NoCutoff);
    bool usePeriodic = (nonbondedMethod == GayBerneForce::CutoffPeriodic);
    map<string, string> defines;
    defines["USE_SWITCH"] = (useCutoff && force.getUseSwitchingFunction() ? "1" : "0");
6665
6666
    double cutoff = force.getCutoffDistance();
    defines["CUTOFF_SQUARED"] = cl.doubleToString(cutoff*cutoff);
6667
    if (useCutoff) {
6668
6669
6670
6671
        defines["USE_CUTOFF"] = 1;
        if (usePeriodic)
            defines["USE_PERIODIC"] = "1";
        
6672
6673
6674
6675
        // Compute the switching coefficients.
        
        if (force.getUseSwitchingFunction()) {
            defines["SWITCH_CUTOFF"] = cl.doubleToString(force.getSwitchingDistance());
6676
6677
6678
            defines["SWITCH_C3"] = cl.doubleToString(10/pow(force.getSwitchingDistance()-cutoff, 3.0));
            defines["SWITCH_C4"] = cl.doubleToString(15/pow(force.getSwitchingDistance()-cutoff, 4.0));
            defines["SWITCH_C5"] = cl.doubleToString(6/pow(force.getSwitchingDistance()-cutoff, 5.0));
6679
6680
        }
    }
6681
    defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
6682
6683
6684
6685
6686
    cl::Program program = cl.createProgram(OpenCLKernelSources::gayBerne, defines);
    framesKernel = cl::Kernel(program, "computeEllipsoidFrames");
    blockBoundsKernel = cl::Kernel(program, "findBlockBounds");
    neighborsKernel = cl::Kernel(program, "findNeighbors");
    forceKernel = cl::Kernel(program, "computeForce");
6687
    torqueKernel = cl::Kernel(program, "applyTorques");
6688
6689
    info = new ForceInfo(cl.getNonbondedUtilities().getNumForceBuffers(), force);
    cl.addForce(info);
6690
6691
6692
6693
6694
6695
6696
6697
6698
    cl.addReorderListener(new ReorderListener(*this));
}

double OpenCLCalcGayBerneForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
        sortAtoms();
        framesKernel.setArg<cl_int>(0, numRealParticles);
        framesKernel.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
6699
6700
6701
6702
6703
6704
6705
        framesKernel.setArg<cl::Buffer>(2, axisParticleIndices.getDeviceBuffer());
        framesKernel.setArg<cl::Buffer>(3, sigParams.getDeviceBuffer());
        framesKernel.setArg<cl::Buffer>(4, scale.getDeviceBuffer());
        framesKernel.setArg<cl::Buffer>(5, aMatrix.getDeviceBuffer());
        framesKernel.setArg<cl::Buffer>(6, bMatrix.getDeviceBuffer());
        framesKernel.setArg<cl::Buffer>(7, gMatrix.getDeviceBuffer());
        framesKernel.setArg<cl::Buffer>(8, sortedParticles.getDeviceBuffer());
6706
        blockBoundsKernel.setArg<cl_int>(0, numRealParticles);
peastman's avatar
peastman committed
6707
        blockBoundsKernel.setArg<cl::Buffer>(6, sortedParticles.getDeviceBuffer());
6708
        blockBoundsKernel.setArg<cl::Buffer>(7, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
6709
6710
6711
6712
        blockBoundsKernel.setArg<cl::Buffer>(8, sortedPos.getDeviceBuffer());
        blockBoundsKernel.setArg<cl::Buffer>(9, blockCenter.getDeviceBuffer());
        blockBoundsKernel.setArg<cl::Buffer>(10, blockBoundingBox.getDeviceBuffer());
        blockBoundsKernel.setArg<cl::Buffer>(11, neighborBlockCount.getDeviceBuffer());
6713
6714
        neighborsKernel.setArg<cl_int>(0, numRealParticles);
        neighborsKernel.setArg<cl_int>(1, maxNeighborBlocks);
peastman's avatar
peastman committed
6715
6716
6717
6718
6719
6720
6721
6722
        neighborsKernel.setArg<cl::Buffer>(7, sortedPos.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(8, blockCenter.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(9, blockBoundingBox.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(10, neighbors.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(11, neighborIndex.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(12, neighborBlockCount.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(13, exclusions.getDeviceBuffer());
        neighborsKernel.setArg<cl::Buffer>(14, exclusionStartIndex.getDeviceBuffer());
6723
        int index = 0;
6724
        forceKernel.setArg<cl::Buffer>(index++, cl.getLongForceBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
6725
        forceKernel.setArg<cl::Buffer>(index++, torque.getDeviceBuffer());
6726
        forceKernel.setArg<cl_int>(index++, numRealParticles);
6727
        forceKernel.setArg<cl_int>(index++, exceptionAtoms.size());
6728
        forceKernel.setArg<cl::Buffer>(index++, cl.getEnergyBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
        forceKernel.setArg<cl::Buffer>(index++, sortedPos.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, sigParams.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, epsParams.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, sortedParticles.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, aMatrix.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, bMatrix.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, gMatrix.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, exclusions.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, exclusionStartIndex.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, exceptionParticles.getDeviceBuffer());
        forceKernel.setArg<cl::Buffer>(index++, exceptionParams.getDeviceBuffer());
6740
6741
        if (nonbondedMethod != GayBerneForce::NoCutoff) {
            forceKernel.setArg<cl_int>(index++, maxNeighborBlocks);
peastman's avatar
peastman committed
6742
6743
6744
            forceKernel.setArg<cl::Buffer>(index++, neighbors.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(index++, neighborIndex.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(index++, neighborBlockCount.getDeviceBuffer());
6745
        }
6746
        index = 0;
6747
        torqueKernel.setArg<cl::Buffer>(index++, cl.getLongForceBuffer().getDeviceBuffer());
peastman's avatar
peastman committed
6748
        torqueKernel.setArg<cl::Buffer>(index++, torque.getDeviceBuffer());
6749
6750
        torqueKernel.setArg<cl_int>(index++, numRealParticles);
        torqueKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
6751
6752
        torqueKernel.setArg<cl::Buffer>(index++, axisParticleIndices.getDeviceBuffer());
        torqueKernel.setArg<cl::Buffer>(index++, sortedParticles.getDeviceBuffer());
6753
6754
    }
    cl.executeKernel(framesKernel, numRealParticles);
6755
6756
    setPeriodicBoxArgs(cl, blockBoundsKernel, 1);
    cl.executeKernel(blockBoundsKernel, (numRealParticles+31)/32);
6757
6758
6759
6760
6761
6762
6763
6764
6765
    if (nonbondedMethod == GayBerneForce::NoCutoff) {
        cl.executeKernel(forceKernel, cl.getNonbondedUtilities().getNumForceThreadBlocks()*cl.getNonbondedUtilities().getForceThreadBlockSize());
    }
    else {
        while (true) {
            setPeriodicBoxArgs(cl, neighborsKernel, 2);
            cl.executeKernel(neighborsKernel, numRealParticles);
            cl_int* count = (cl_int*) cl.getPinnedBuffer();
            cl::Event event;
peastman's avatar
peastman committed
6766
            cl.getQueue().enqueueReadBuffer(neighborBlockCount.getDeviceBuffer(), CL_FALSE, 0, neighborBlockCount.getSize()*neighborBlockCount.getElementSize(), count, NULL, &event);
6767
6768
6769
6770
6771
6772
6773
6774
6775
            setPeriodicBoxArgs(cl, forceKernel, 20);
            cl.executeKernel(forceKernel, cl.getNonbondedUtilities().getNumForceThreadBlocks()*cl.getNonbondedUtilities().getForceThreadBlockSize());
            event.wait();
            if (*count <= maxNeighborBlocks)
                break;
            
            // There wasn't enough room for the neighbor list, so we need to recreate it.

            maxNeighborBlocks = (int) ceil((*count)*1.1);
peastman's avatar
peastman committed
6776
6777
6778
6779
6780
6781
            neighbors.resize(maxNeighborBlocks*32);
            neighborIndex.resize(maxNeighborBlocks);
            neighborsKernel.setArg<cl::Buffer>(10, neighbors.getDeviceBuffer());
            neighborsKernel.setArg<cl::Buffer>(11, neighborIndex.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(17, neighbors.getDeviceBuffer());
            forceKernel.setArg<cl::Buffer>(18, neighborIndex.getDeviceBuffer());
6782
        }
6783
    }
6784
    cl.executeKernel(torqueKernel, numRealParticles);
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
    return 0.0;
}

void OpenCLCalcGayBerneForceKernel::copyParametersToContext(ContextImpl& context, const GayBerneForce& force) {
    // Make sure the new parameters are acceptable.
    
    if (force.getNumParticles() != cl.getNumAtoms())
        throw OpenMMException("updateParametersInContext: The number of particles has changed");
    vector<int> exceptions;
    for (int i = 0; i < force.getNumExceptions(); i++) {
        int particle1, particle2;
        double sigma, epsilon;
        force.getExceptionParameters(i, particle1, particle2, sigma, epsilon);
        if (exceptionAtoms.size() > exceptions.size() && make_pair(particle1, particle2) == exceptionAtoms[exceptions.size()])
            exceptions.push_back(i);
        else if (epsilon != 0.0)
            throw OpenMMException("updateParametersInContext: The set of non-excluded exceptions has changed");
    }
6803
    int numExceptions = exceptionAtoms.size();
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
    
    // Record the per-particle parameters.
    
    vector<mm_float4> sigParamsVector(cl.getPaddedNumAtoms(), mm_float4(0, 0, 0, 0));
    vector<mm_float2> epsParamsVector(cl.getPaddedNumAtoms(), mm_float2(0, 0));
    vector<mm_float4> scaleVector(cl.getPaddedNumAtoms(), mm_float4(0, 0, 0, 0));
    for (int i = 0; i < force.getNumParticles(); i++) {
        int xparticle, yparticle;
        double sigma, epsilon, sx, sy, sz, ex, ey, ez;
        force.getParticleParameters(i, sigma, epsilon, xparticle, yparticle, sx, sy, sz, ex, ey, ez);
6814
        sigParamsVector[i] = mm_float4((float) (0.5*sigma), (float) (0.25*sx*sx), (float) (0.25*sy*sy), (float) (0.25*sz*sz));
6815
6816
        epsParamsVector[i] = mm_float2((float) sqrt(epsilon), (float) (0.125*(sx*sy + sz*sz)*sqrt(sx*sy)));
        scaleVector[i] = mm_float4((float) (1/sqrt(ex)), (float) (1/sqrt(ey)), (float) (1/sqrt(ez)), 0);
6817
6818
        if (epsilon != 0.0 && !isRealParticle[i])
            throw OpenMMException("updateParametersInContext: The set of ignored particles (ones with epsilon=0) has changed");
6819
    }
peastman's avatar
peastman committed
6820
6821
6822
    sigParams.upload(sigParamsVector);
    epsParams.upload(epsParamsVector);
    scale.upload(scaleVector);
6823
6824
6825
6826
    
    // Record the exceptions.
    
    if (numExceptions > 0) {
6827
        vector<mm_float2> exceptionParamsVec(numExceptions);
6828
        for (int i = 0; i < numExceptions; i++) {
6829
            int atom1, atom2;
6830
            double sigma, epsilon;
6831
6832
            force.getExceptionParameters(exceptions[i], atom1, atom2, sigma, epsilon);
            exceptionParamsVec[i] = mm_float2((float) sigma, (float) epsilon);
6833
        }
peastman's avatar
peastman committed
6834
        exceptionParams.upload(exceptionParamsVec);
6835
    }
6836
    cl.invalidateMolecules(info);
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
    sortAtoms();
}

void OpenCLCalcGayBerneForceKernel::sortAtoms() {
    // Sort the list of atoms by type to avoid thread divergence.  This is executed every time
    // the atoms are reordered.
    
    int nextIndex = 0;
    vector<cl_int> particles(cl.getPaddedNumAtoms(), 0);
    const vector<int>& order = cl.getAtomIndex();
6847
    vector<int> inverseOrder(order.size(), -1);
6848
6849
    for (int i = 0; i < cl.getNumAtoms(); i++) {
        int atom = order[i];
6850
6851
        if (isRealParticle[atom]) {
            inverseOrder[atom] = nextIndex;
6852
            particles[nextIndex++] = atom;
6853
        }
6854
    }
peastman's avatar
peastman committed
6855
    sortedParticles.upload(particles);
6856
    
6857
6858
6859
6860
6861
6862
6863
    // Update the list of exception particles.
    
    int numExceptions = exceptionAtoms.size();
    if (numExceptions > 0) {
        vector<mm_int4> exceptionParticlesVec(numExceptions);
        for (int i = 0; i < numExceptions; i++)
            exceptionParticlesVec[i] = mm_int4(exceptionAtoms[i].first, exceptionAtoms[i].second, inverseOrder[exceptionAtoms[i].first], inverseOrder[exceptionAtoms[i].second]);
peastman's avatar
peastman committed
6864
        exceptionParticles.upload(exceptionParticlesVec);
6865
6866
    }
    
6867
6868
6869
6870
    // Rebuild the list of exclusions.
    
    vector<vector<int> > excludedAtoms(numRealParticles);
    for (int i = 0; i < excludedPairs.size(); i++) {
6871
6872
        int first = inverseOrder[min(excludedPairs[i].first, excludedPairs[i].second)];
        int second = inverseOrder[max(excludedPairs[i].first, excludedPairs[i].second)];
6873
6874
6875
        excludedAtoms[first].push_back(second);
    }
    int index = 0;
peastman's avatar
peastman committed
6876
6877
    vector<int> exclusionVec(exclusions.getSize());
    vector<int> startIndexVec(exclusionStartIndex.getSize());
6878
6879
6880
6881
6882
6883
    for (int i = 0; i < numRealParticles; i++) {
        startIndexVec[i] = index;
        for (int j = 0; j < excludedAtoms[i].size(); j++)
            exclusionVec[index++] = excludedAtoms[i][j];
    }
    startIndexVec[numRealParticles] = index;
peastman's avatar
peastman committed
6884
6885
    exclusions.upload(exclusionVec);
    exclusionStartIndex.upload(startIndexVec);
6886
6887
}

6888
6889
6890
6891
6892
class OpenCLCalcCustomCVForceKernel::ReorderListener : public OpenCLContext::ReorderListener {
public:
    ReorderListener(OpenCLContext& cl, OpenCLArray& invAtomOrder) : cl(cl), invAtomOrder(invAtomOrder) {
    }
    void execute() {
6893
        vector<cl_int> invOrder(cl.getPaddedNumAtoms());
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
        const vector<int>& order = cl.getAtomIndex();
        for (int i = 0; i < order.size(); i++)
            invOrder[order[i]] = i;
        invAtomOrder.upload(invOrder);
    }
private:
    OpenCLContext& cl;
    OpenCLArray& invAtomOrder;
};

6904
void OpenCLCalcCustomCVForceKernel::initialize(const System& system, const CustomCVForce& force, ContextImpl& innerContext) {
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
    int numCVs = force.getNumCollectiveVariables();
    cl.addForce(new OpenCLForceInfo(1));
    for (int i = 0; i < force.getNumGlobalParameters(); i++)
        globalParameterNames.push_back(force.getGlobalParameterName(i));
    
    // Create custom functions for the tabulated functions.

    map<string, Lepton::CustomFunction*> functions;
    for (int i = 0; i < (int) force.getNumTabulatedFunctions(); i++)
        functions[force.getTabulatedFunctionName(i)] = createReferenceTabulatedFunction(force.getTabulatedFunction(i));

    // Create the expressions.

    Lepton::ParsedExpression energyExpr = Lepton::Parser::parse(force.getEnergyFunction(), functions);
    energyExpression = energyExpr.createProgram();
    for (int i = 0; i < numCVs; i++) {
        string name = force.getCollectiveVariableName(i);
        variableNames.push_back(name);
        variableDerivExpressions.push_back(energyExpr.differentiate(name).optimize().createProgram());
    }
    for (int i = 0; i < force.getNumEnergyParameterDerivatives(); i++) {
        string name = force.getEnergyParameterDerivativeName(i);
        paramDerivNames.push_back(name);
        paramDerivExpressions.push_back(energyExpr.differentiate(name).optimize().createProgram());
6929
        cl.addEnergyParameterDerivative(name);
6930
6931
6932
6933
6934
6935
    }

    // Delete the custom functions.

    for (auto& function : functions)
        delete function.second;
6936
6937
6938
6939
6940
6941
        
    // Copy parameter derivatives from the inner context.

    OpenCLContext& cl2 = *reinterpret_cast<OpenCLPlatform::PlatformData*>(innerContext.getPlatformData())->contexts[0];
    for (auto& param : cl2.getEnergyParamDerivNames())
        cl.addEnergyParameterDerivative(param);
6942
6943
6944
6945
    
    // Create arrays for storing information.
    
    int elementSize = (cl.getUseDoublePrecision() || cl.getUseMixedPrecision() ? sizeof(double) : sizeof(float));
peastman's avatar
peastman committed
6946
    cvForces.resize(numCVs);
6947
    for (int i = 0; i < numCVs; i++)
peastman's avatar
peastman committed
6948
6949
6950
        cvForces[i].initialize(cl, cl.getNumAtoms(), 4*elementSize, "cvForce");
    invAtomOrder.initialize<cl_int>(cl, cl.getPaddedNumAtoms(), "invAtomOrder");
    innerInvAtomOrder.initialize<cl_int>(cl, cl.getPaddedNumAtoms(), "innerInvAtomOrder");
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
    
    // Create the kernels.
    
    stringstream args, add;
    for (int i = 0; i < numCVs; i++) {
        args << ", __global real4* restrict force" << i << ", real dEdV" << i;
        add << "f += force" << i << "[i]*dEdV" << i << ";\n";
    }
    map<string, string> replacements;
    replacements["PARAMETER_ARGUMENTS"] = args.str();
    replacements["ADD_FORCES"] = add.str();
    cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customCVForce, replacements));
    copyStateKernel = cl::Kernel(program, "copyState");
    copyForcesKernel = cl::Kernel(program, "copyForces");
    addForcesKernel = cl::Kernel(program, "addForces");
}

double OpenCLCalcCustomCVForceKernel::execute(ContextImpl& context, ContextImpl& innerContext, bool includeForces, bool includeEnergy) {
    copyState(context, innerContext);
    int numCVs = variableNames.size();
    int numAtoms = cl.getNumAtoms();
    OpenCLContext& cl2 = *reinterpret_cast<OpenCLPlatform::PlatformData*>(innerContext.getPlatformData())->contexts[0];
    vector<double> cvValues;
    vector<map<string, double> > cvDerivs(numCVs);
    for (int i = 0; i < numCVs; i++) {
        cvValues.push_back(innerContext.calcForcesAndEnergy(true, true, 1<<i));
peastman's avatar
peastman committed
6977
        copyForcesKernel.setArg<cl::Buffer>(0, cvForces[i].getDeviceBuffer());
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
        cl.executeKernel(copyForcesKernel, numAtoms);
        innerContext.getEnergyParameterDerivatives(cvDerivs[i]);
    }
    
    // Compute the energy and forces.
    
    map<string, double> variables;
    for (auto& name : globalParameterNames)
        variables[name] = context.getParameter(name);
    for (int i = 0; i < numCVs; i++)
        variables[variableNames[i]] = cvValues[i];
    double energy = energyExpression.evaluate(variables);
    for (int i = 0; i < numCVs; i++) {
        double dEdV = variableDerivExpressions[i].evaluate(variables);
        if (cl.getUseDoublePrecision())
            addForcesKernel.setArg<cl_double>(2*i+3, dEdV);
        else
            addForcesKernel.setArg<cl_float>(2*i+3, dEdV);
    }
6997
    cl.executeKernel(addForcesKernel, numAtoms);
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
    
    // Compute the energy parameter derivatives.
    
    map<string, double>& energyParamDerivs = cl.getEnergyParamDerivWorkspace();
    for (int i = 0; i < paramDerivExpressions.size(); i++)
        energyParamDerivs[paramDerivNames[i]] += paramDerivExpressions[i].evaluate(variables);
    for (int i = 0; i < numCVs; i++) {
        double dEdV = variableDerivExpressions[i].evaluate(variables);
        for (auto& deriv : cvDerivs[i])
            energyParamDerivs[deriv.first] += dEdV*deriv.second;
    }
    return energy;
}

void OpenCLCalcCustomCVForceKernel::copyState(ContextImpl& context, ContextImpl& innerContext) {
    int numAtoms = cl.getNumAtoms();
7014
    OpenCLContext& cl2 = *reinterpret_cast<OpenCLPlatform::PlatformData*>(innerContext.getPlatformData())->contexts[0];
7015
7016
7017
7018
7019
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
        
        // Initialize the listeners.
        
peastman's avatar
peastman committed
7020
7021
        ReorderListener* listener1 = new ReorderListener(cl, invAtomOrder);
        ReorderListener* listener2 = new ReorderListener(cl2, innerInvAtomOrder);
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
        cl.addReorderListener(listener1);
        cl2.addReorderListener(listener2);
        listener1->execute();
        listener2->execute();
        
        // Initialize the kernels.
        
        copyStateKernel.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
        copyStateKernel.setArg<cl::Buffer>(2, cl.getVelm().getDeviceBuffer());
        copyStateKernel.setArg<cl::Buffer>(3, cl.getAtomIndexArray().getDeviceBuffer());
        copyStateKernel.setArg<cl::Buffer>(4, cl2.getPosq().getDeviceBuffer());
        copyStateKernel.setArg<cl::Buffer>(6, cl2.getVelm().getDeviceBuffer());
peastman's avatar
peastman committed
7034
        copyStateKernel.setArg<cl::Buffer>(7, innerInvAtomOrder.getDeviceBuffer());
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
        copyStateKernel.setArg<cl_int>(8, numAtoms);
        if (cl.getUseMixedPrecision()) {
            copyStateKernel.setArg<cl::Buffer>(1, cl.getPosqCorrection().getDeviceBuffer());
            copyStateKernel.setArg<cl::Buffer>(5, cl2.getPosqCorrection().getDeviceBuffer());
        }
        else {
            copyStateKernel.setArg<void*>(1, NULL);
            copyStateKernel.setArg<void*>(5, NULL);
        }

peastman's avatar
peastman committed
7045
        copyForcesKernel.setArg<cl::Buffer>(1, invAtomOrder.getDeviceBuffer());
7046
7047
7048
7049
7050
7051
7052
        copyForcesKernel.setArg<cl::Buffer>(2, cl2.getForce().getDeviceBuffer());
        copyForcesKernel.setArg<cl::Buffer>(3, cl2.getAtomIndexArray().getDeviceBuffer());
        copyForcesKernel.setArg<cl_int>(4, numAtoms);

        addForcesKernel.setArg<cl::Buffer>(0, cl.getForce().getDeviceBuffer());
        addForcesKernel.setArg<cl_int>(1, numAtoms);
        for (int i = 0; i < cvForces.size(); i++)
peastman's avatar
peastman committed
7053
            addForcesKernel.setArg<cl::Buffer>(2*i+2, cvForces[i].getDeviceBuffer());
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
    }
    cl.executeKernel(copyStateKernel, numAtoms);
    Vec3 a, b, c;
    context.getPeriodicBoxVectors(a, b, c);
    innerContext.setPeriodicBoxVectors(a, b, c);
    innerContext.setTime(context.getTime());
    map<string, double> innerParameters = innerContext.getParameters();
    for (auto& param : innerParameters)
        innerContext.setParameter(param.first, context.getParameter(param.first));
}

peastman's avatar
peastman committed
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
class OpenCLCalcRMSDForceKernel::ForceInfo : public OpenCLForceInfo {
public:
    ForceInfo(const RMSDForce& force) : OpenCLForceInfo(0), force(force) {
        updateParticles();
    }
    void updateParticles() {
        particles.clear();
        for (int i : force.getParticles())
            particles.insert(i);
    }
    bool areParticlesIdentical(int particle1, int particle2) {
        bool include1 = (particles.find(particle1) != particles.end());
        bool include2 = (particles.find(particle2) != particles.end());
        return (include1 == include2);
    }
private:
    const RMSDForce& force;
    set<int> particles;
};

void OpenCLCalcRMSDForceKernel::initialize(const System& system, const RMSDForce& force) {
    // Create data structures.
    
    bool useDouble = cl.getUseDoublePrecision();
    int elementSize = (useDouble ? sizeof(cl_double) : sizeof(cl_float));
    int numParticles = force.getParticles().size();
    if (numParticles == 0)
        numParticles = system.getNumParticles();
peastman's avatar
peastman committed
7093
7094
7095
    referencePos.initialize(cl, system.getNumParticles(), 4*elementSize, "referencePos");
    particles.initialize<cl_int>(cl, numParticles, "particles");
    buffer.initialize(cl, 13, elementSize, "buffer");
peastman's avatar
peastman committed
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
    recordParameters(force);
    info = new ForceInfo(force);
    cl.addForce(info);
    
    // Create the kernels.

    cl::Program program = cl.createProgram(OpenCLKernelSources::rmsd);
    kernel1 = cl::Kernel(program, "computeRMSDPart1");
    kernel2 = cl::Kernel(program, "computeRMSDForces");
}

void OpenCLCalcRMSDForceKernel::recordParameters(const RMSDForce& force) {
    // Record the parameters and center the reference positions.
    
    vector<int> particleVec = force.getParticles();
    if (particleVec.size() == 0)
        for (int i = 0; i < cl.getNumAtoms(); i++)
            particleVec.push_back(i);
    vector<Vec3> centeredPositions = force.getReferencePositions();
    Vec3 center;
    for (int i : particleVec)
        center += centeredPositions[i];
    center /= particleVec.size();
    for (Vec3& p : centeredPositions)
        p -= center;

    // Upload them to the device.

peastman's avatar
peastman committed
7124
    particles.upload(particleVec);
peastman's avatar
peastman committed
7125
7126
7127
7128
    if (cl.getUseDoublePrecision()) {
        vector<mm_double4> pos;
        for (Vec3 p : centeredPositions)
            pos.push_back(mm_double4(p[0], p[1], p[2], 0));
peastman's avatar
peastman committed
7129
        referencePos.upload(pos);
peastman's avatar
peastman committed
7130
7131
7132
7133
7134
    }
    else {
        vector<mm_float4> pos;
        for (Vec3 p : centeredPositions)
            pos.push_back(mm_float4(p[0], p[1], p[2], 0));
peastman's avatar
peastman committed
7135
        referencePos.upload(pos);
peastman's avatar
peastman committed
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
    }

    // Record the sum of the norms of the reference positions.

    sumNormRef = 0.0;
    for (int i : particleVec) {
        Vec3 p = centeredPositions[i];
        sumNormRef += p.dot(p);
    }
}

double OpenCLCalcRMSDForceKernel::execute(ContextImpl& context, bool includeForces, bool includeEnergy) {
    if (cl.getUseDoublePrecision())
        return executeImpl<double>(context);
    return executeImpl<float>(context);
}

template <class REAL>
double OpenCLCalcRMSDForceKernel::executeImpl(ContextImpl& context) {
    // Execute the first kernel.

peastman's avatar
peastman committed
7157
    int numParticles = particles.getSize();
Peter Eastman's avatar
Peter Eastman committed
7158
    int blockSize = min(256, (int) kernel1.getWorkGroupInfo<CL_KERNEL_WORK_GROUP_SIZE>(cl.getDevice()));
peastman's avatar
peastman committed
7159
7160
    kernel1.setArg<cl_int>(0, numParticles);
    kernel1.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
7161
7162
7163
    kernel1.setArg<cl::Buffer>(2, referencePos.getDeviceBuffer());
    kernel1.setArg<cl::Buffer>(3, particles.getDeviceBuffer());
    kernel1.setArg<cl::Buffer>(4, buffer.getDeviceBuffer());
peastman's avatar
peastman committed
7164
7165
7166
7167
7168
7169
7170
    kernel1.setArg(5, blockSize*sizeof(REAL), NULL);
    cl.executeKernel(kernel1, blockSize, blockSize);
    
    // Download the results, build the F matrix, and find the maximum eigenvalue
    // and eigenvector.

    vector<REAL> b;
peastman's avatar
peastman committed
7171
    buffer.download(b);
peastman's avatar
peastman committed
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
    Array2D<double> F(4, 4);
    F[0][0] =  b[0*3+0] + b[1*3+1] + b[2*3+2];
    F[1][0] =  b[1*3+2] - b[2*3+1];
    F[2][0] =  b[2*3+0] - b[0*3+2];
    F[3][0] =  b[0*3+1] - b[1*3+0];
    F[0][1] =  b[1*3+2] - b[2*3+1];
    F[1][1] =  b[0*3+0] - b[1*3+1] - b[2*3+2];
    F[2][1] =  b[0*3+1] + b[1*3+0];
    F[3][1] =  b[0*3+2] + b[2*3+0];
    F[0][2] =  b[2*3+0] - b[0*3+2];
    F[1][2] =  b[0*3+1] + b[1*3+0];
    F[2][2] = -b[0*3+0] + b[1*3+1] - b[2*3+2];
    F[3][2] =  b[1*3+2] + b[2*3+1];
    F[0][3] =  b[0*3+1] - b[1*3+0];
    F[1][3] =  b[0*3+2] + b[2*3+0];
    F[2][3] =  b[1*3+2] + b[2*3+1];
    F[3][3] = -b[0*3+0] - b[1*3+1] + b[2*3+2];
    JAMA::Eigenvalue<double> eigen(F);
    Array1D<double> values;
    eigen.getRealEigenvalues(values);
    Array2D<double> vectors;
    eigen.getV(vectors);

    // Compute the RMSD.

    double msd = (sumNormRef+b[9]-2*values[3])/numParticles;
7198
7199
7200
7201
7202
    if (msd < 1e-20) {
        // The particles are perfectly aligned, so all the forces should be zero.
        // Numerical error can lead to NaNs, so just return 0 now.
        return 0.0;
    }
peastman's avatar
peastman committed
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
    double rmsd = sqrt(msd);
    b[9] = rmsd;

    // Compute the rotation matrix.

    double q[] = {vectors[0][3], vectors[1][3], vectors[2][3], vectors[3][3]};
    double q00 = q[0]*q[0], q01 = q[0]*q[1], q02 = q[0]*q[2], q03 = q[0]*q[3];
    double q11 = q[1]*q[1], q12 = q[1]*q[2], q13 = q[1]*q[3];
    double q22 = q[2]*q[2], q23 = q[2]*q[3];
    double q33 = q[3]*q[3];
    b[0] = q00+q11-q22-q33;
    b[1] = 2*(q12-q03);
    b[2] = 2*(q13+q02);
    b[3] = 2*(q12+q03);
    b[4] = q00-q11+q22-q33;
    b[5] = 2*(q23-q01);
    b[6] = 2*(q13-q02);
    b[7] = 2*(q23+q01);
    b[8] = q00-q11-q22+q33;

    // Upload it to the device and invoke the kernel to apply forces.
    
peastman's avatar
peastman committed
7225
    buffer.upload(b);
peastman's avatar
peastman committed
7226
7227
    kernel2.setArg<cl_int>(0, numParticles);
    kernel2.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
7228
7229
7230
    kernel2.setArg<cl::Buffer>(2, referencePos.getDeviceBuffer());
    kernel2.setArg<cl::Buffer>(3, particles.getDeviceBuffer());
    kernel2.setArg<cl::Buffer>(4, buffer.getDeviceBuffer());
peastman's avatar
peastman committed
7231
7232
7233
7234
7235
7236
    kernel2.setArg<cl::Buffer>(5, cl.getForceBuffers().getDeviceBuffer());
    cl.executeKernel(kernel2, numParticles);
    return rmsd;
}

void OpenCLCalcRMSDForceKernel::copyParametersToContext(ContextImpl& context, const RMSDForce& force) {
peastman's avatar
peastman committed
7237
    if (referencePos.getSize() != force.getReferencePositions().size())
peastman's avatar
peastman committed
7238
7239
7240
7241
        throw OpenMMException("updateParametersInContext: The number of reference positions has changed");
    int numParticles = force.getParticles().size();
    if (numParticles == 0)
        numParticles = context.getSystem().getNumParticles();
peastman's avatar
peastman committed
7242
7243
    if (numParticles != particles.getSize())
        particles.resize(numParticles);
peastman's avatar
peastman committed
7244
7245
7246
7247
7248
7249
7250
7251
    recordParameters(force);
    
    // Mark that the current reordering may be invalid.
    
    info->updateParticles();
    cl.invalidateMolecules(info);
}

7252
7253
7254
7255
OpenCLIntegrateVerletStepKernel::~OpenCLIntegrateVerletStepKernel() {
}

void OpenCLIntegrateVerletStepKernel::initialize(const System& system, const VerletIntegrator& integrator) {
7256
    cl.getPlatformData().initializeContexts(system);
7257
    cl::Program program = cl.createProgram(OpenCLKernelSources::verlet, "");
7258
7259
    kernel1 = cl::Kernel(program, "integrateVerletPart1");
    kernel2 = cl::Kernel(program, "integrateVerletPart2");
7260
7261
7262
}

void OpenCLIntegrateVerletStepKernel::execute(ContextImpl& context, const VerletIntegrator& integrator) {
7263
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
7264
7265
    int numAtoms = cl.getNumAtoms();
    double dt = integrator.getStepSize();
7266
7267
7268
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
        kernel1.setArg<cl_int>(0, numAtoms);
7269
        kernel1.setArg<cl::Buffer>(1, cl.getIntegrationUtilities().getStepSize().getDeviceBuffer());
7270
        kernel1.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
7271
7272
7273
7274
        setPosqCorrectionArg(cl, kernel1, 3);
        kernel1.setArg<cl::Buffer>(4, cl.getVelm().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(5, cl.getForce().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(6, integration.getPosDelta().getDeviceBuffer());
7275
        kernel2.setArg<cl_int>(0, numAtoms);
7276
        kernel2.setArg<cl::Buffer>(1, cl.getIntegrationUtilities().getStepSize().getDeviceBuffer());
7277
        kernel2.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
7278
7279
7280
        setPosqCorrectionArg(cl, kernel2, 3);
        kernel2.setArg<cl::Buffer>(4, cl.getVelm().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(5, integration.getPosDelta().getDeviceBuffer());
7281
    }
7282
    cl.getIntegrationUtilities().setNextStepSize(dt);
7283
7284
7285
7286
7287
7288
7289

    // Call the first integration kernel.

    cl.executeKernel(kernel1, numAtoms);

    // Apply constraints.

7290
    integration.applyConstraints(integrator.getConstraintTolerance());
7291
7292
7293
7294

    // Call the second integration kernel.

    cl.executeKernel(kernel2, numAtoms);
7295
    integration.computeVirtualSites();
7296
7297
7298
7299
7300

    // Update the time and step count.

    cl.setTime(cl.getTime()+dt);
    cl.setStepCount(cl.getStepCount()+1);
7301
    cl.reorderAtoms();
7302
7303
7304
7305
7306
7307
    
    // Reduce UI lag.
    
#ifdef WIN32
    cl.getQueue().flush();
#endif
7308
7309
}

7310
7311
7312
7313
double OpenCLIntegrateVerletStepKernel::computeKineticEnergy(ContextImpl& context, const VerletIntegrator& integrator) {
    return cl.getIntegrationUtilities().computeKineticEnergy(0.5*integrator.getStepSize());
}

7314
void OpenCLIntegrateLangevinStepKernel::initialize(const System& system, const LangevinIntegrator& integrator) {
7315
    cl.getPlatformData().initializeContexts(system);
7316
7317
    cl.getIntegrationUtilities().initRandomNumberGenerator(integrator.getRandomNumberSeed());
    map<string, string> defines;
7318
7319
    defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
    defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
7320
    cl::Program program = cl.createProgram(OpenCLKernelSources::langevin, defines, "");
7321
7322
    kernel1 = cl::Kernel(program, "integrateLangevinPart1");
    kernel2 = cl::Kernel(program, "integrateLangevinPart2");
peastman's avatar
peastman committed
7323
    params.initialize(cl, 3, cl.getUseDoublePrecision() || cl.getUseMixedPrecision() ? sizeof(cl_double) : sizeof(cl_float), "langevinParams");
7324
7325
7326
7327
    prevStepSize = -1.0;
}

void OpenCLIntegrateLangevinStepKernel::execute(ContextImpl& context, const LangevinIntegrator& integrator) {
7328
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
7329
    int numAtoms = cl.getNumAtoms();
7330
7331
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
7332
7333
7334
        kernel1.setArg<cl::Buffer>(0, cl.getVelm().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(1, cl.getForce().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(2, integration.getPosDelta().getDeviceBuffer());
peastman's avatar
peastman committed
7335
        kernel1.setArg<cl::Buffer>(3, params.getDeviceBuffer());
7336
7337
7338
        kernel1.setArg<cl::Buffer>(4, integration.getStepSize().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(5, integration.getRandom().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
7339
7340
7341
7342
        setPosqCorrectionArg(cl, kernel2, 1);
        kernel2.setArg<cl::Buffer>(2, integration.getPosDelta().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(3, cl.getVelm().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(4, integration.getStepSize().getDeviceBuffer());
7343
    }
7344
7345
7346
    double temperature = integrator.getTemperature();
    double friction = integrator.getFriction();
    double stepSize = integrator.getStepSize();
7347
    cl.getIntegrationUtilities().setNextStepSize(stepSize);
7348
7349
7350
7351
    if (temperature != prevTemp || friction != prevFriction || stepSize != prevStepSize) {
        // Calculate the integration parameters.

        double kT = BOLTZ*temperature;
7352
7353
7354
        double vscale = exp(-stepSize*friction);
        double fscale = (friction == 0 ? stepSize : (1-vscale)/friction);
        double noisescale = sqrt(kT*(1-vscale*vscale));
7355
        if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
peastman's avatar
peastman committed
7356
            vector<cl_double> p(params.getSize());
7357
7358
7359
            p[0] = vscale;
            p[1] = fscale;
            p[2] = noisescale;
peastman's avatar
peastman committed
7360
            params.upload(p);
7361
7362
        }
        else {
peastman's avatar
peastman committed
7363
            vector<cl_float> p(params.getSize());
7364
7365
7366
            p[0] = (cl_float) vscale;
            p[1] = (cl_float) fscale;
            p[2] = (cl_float) noisescale;
peastman's avatar
peastman committed
7367
            params.upload(p);
7368
        }
7369
7370
7371
7372
7373
7374
7375
        prevTemp = temperature;
        prevFriction = friction;
        prevStepSize = stepSize;
    }

    // Call the first integration kernel.

7376
    kernel1.setArg<cl_uint>(6, integration.prepareRandomNumbers(cl.getPaddedNumAtoms()));
7377
7378
7379
7380
    cl.executeKernel(kernel1, numAtoms);

    // Apply constraints.

7381
    integration.applyConstraints(integrator.getConstraintTolerance());
7382
7383
7384
7385

    // Call the second integration kernel.

    cl.executeKernel(kernel2, numAtoms);
7386
    integration.computeVirtualSites();
7387
7388
7389
7390
7391

    // Update the time and step count.

    cl.setTime(cl.getTime()+stepSize);
    cl.setStepCount(cl.getStepCount()+1);
7392
    cl.reorderAtoms();
7393
7394
7395
7396
7397
7398
    
    // Reduce UI lag.
    
#ifdef WIN32
    cl.getQueue().flush();
#endif
7399
}
7400

7401
7402
7403
7404
double OpenCLIntegrateLangevinStepKernel::computeKineticEnergy(ContextImpl& context, const LangevinIntegrator& integrator) {
    return cl.getIntegrationUtilities().computeKineticEnergy(0.5*integrator.getStepSize());
}

7405
7406
7407
7408
OpenCLIntegrateBrownianStepKernel::~OpenCLIntegrateBrownianStepKernel() {
}

void OpenCLIntegrateBrownianStepKernel::initialize(const System& system, const BrownianIntegrator& integrator) {
7409
    cl.getPlatformData().initializeContexts(system);
7410
7411
    cl.getIntegrationUtilities().initRandomNumberGenerator(integrator.getRandomNumberSeed());
    map<string, string> defines;
7412
    defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
7413
    cl::Program program = cl.createProgram(OpenCLKernelSources::brownian, defines, "");
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
    kernel1 = cl::Kernel(program, "integrateBrownianPart1");
    kernel2 = cl::Kernel(program, "integrateBrownianPart2");
    prevStepSize = -1.0;
}

void OpenCLIntegrateBrownianStepKernel::execute(ContextImpl& context, const BrownianIntegrator& integrator) {
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
    int numAtoms = cl.getNumAtoms();
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
        kernel1.setArg<cl::Buffer>(2, cl.getForce().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(3, integration.getPosDelta().getDeviceBuffer());
7426
7427
        kernel1.setArg<cl::Buffer>(4, cl.getVelm().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(5, integration.getRandom().getDeviceBuffer());
7428
        kernel2.setArg<cl::Buffer>(1, cl.getPosq().getDeviceBuffer());
7429
7430
7431
        setPosqCorrectionArg(cl, kernel2, 2);
        kernel2.setArg<cl::Buffer>(3, cl.getVelm().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(4, integration.getPosDelta().getDeviceBuffer());
7432
7433
7434
7435
7436
7437
    }
    double temperature = integrator.getTemperature();
    double friction = integrator.getFriction();
    double stepSize = integrator.getStepSize();
    if (temperature != prevTemp || friction != prevFriction || stepSize != prevStepSize) {
        double tau = (friction == 0.0 ? 0.0 : 1.0/friction);
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
        if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
            kernel1.setArg<cl_double>(0, tau*stepSize);
            kernel1.setArg<cl_double>(1, sqrt(2.0f*BOLTZ*temperature*stepSize*tau));
            kernel2.setArg<cl_double>(0, 1.0/stepSize);
        }
        else {
            kernel1.setArg<cl_float>(0, (cl_float) (tau*stepSize));
            kernel1.setArg<cl_float>(1, (cl_float) (sqrt(2.0f*BOLTZ*temperature*stepSize*tau)));
            kernel2.setArg<cl_float>(0, (cl_float) (1.0/stepSize));
        }
7448
7449
7450
7451
7452
7453
7454
        prevTemp = temperature;
        prevFriction = friction;
        prevStepSize = stepSize;
    }

    // Call the first integration kernel.

7455
    kernel1.setArg<cl_uint>(6, integration.prepareRandomNumbers(cl.getPaddedNumAtoms()));
7456
7457
7458
7459
7460
7461
7462
7463
7464
    cl.executeKernel(kernel1, numAtoms);

    // Apply constraints.

    integration.applyConstraints(integrator.getConstraintTolerance());

    // Call the second integration kernel.

    cl.executeKernel(kernel2, numAtoms);
7465
    integration.computeVirtualSites();
7466
7467
7468
7469
7470

    // Update the time and step count.

    cl.setTime(cl.getTime()+stepSize);
    cl.setStepCount(cl.getStepCount()+1);
7471
    cl.reorderAtoms();
7472
7473
7474
7475
7476
7477
    
    // Reduce UI lag.
    
#ifdef WIN32
    cl.getQueue().flush();
#endif
7478
}
7479

7480
7481
7482
7483
double OpenCLIntegrateBrownianStepKernel::computeKineticEnergy(ContextImpl& context, const BrownianIntegrator& integrator) {
    return cl.getIntegrationUtilities().computeKineticEnergy(0);
}

7484
7485
7486
7487
OpenCLIntegrateVariableVerletStepKernel::~OpenCLIntegrateVariableVerletStepKernel() {
}

void OpenCLIntegrateVariableVerletStepKernel::initialize(const System& system, const VariableVerletIntegrator& integrator) {
7488
    cl.getPlatformData().initializeContexts(system);
7489
    cl::Program program = cl.createProgram(OpenCLKernelSources::verlet, "");
7490
7491
7492
    kernel1 = cl::Kernel(program, "integrateVerletPart1");
    kernel2 = cl::Kernel(program, "integrateVerletPart2");
    selectSizeKernel = cl::Kernel(program, "selectVerletStepSize");
7493
    blockSize = min(min(256, system.getNumParticles()), (int) selectSizeKernel.getWorkGroupInfo<CL_KERNEL_WORK_GROUP_SIZE>(cl.getDevice()));
7494
7495
}

7496
double OpenCLIntegrateVariableVerletStepKernel::execute(ContextImpl& context, const VariableVerletIntegrator& integrator, double maxTime) {
7497
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
7498
    int numAtoms = cl.getNumAtoms();
7499
    bool useDouble = cl.getUseDoublePrecision() || cl.getUseMixedPrecision();
7500
7501
7502
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
        kernel1.setArg<cl_int>(0, numAtoms);
7503
        kernel1.setArg<cl::Buffer>(1, cl.getIntegrationUtilities().getStepSize().getDeviceBuffer());
7504
        kernel1.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
7505
7506
7507
7508
        setPosqCorrectionArg(cl, kernel1, 3);
        kernel1.setArg<cl::Buffer>(4, cl.getVelm().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(5, cl.getForce().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(6, integration.getPosDelta().getDeviceBuffer());
7509
        kernel2.setArg<cl_int>(0, numAtoms);
7510
        kernel2.setArg<cl::Buffer>(1, cl.getIntegrationUtilities().getStepSize().getDeviceBuffer());
7511
        kernel2.setArg<cl::Buffer>(2, cl.getPosq().getDeviceBuffer());
7512
7513
7514
        setPosqCorrectionArg(cl, kernel2, 3);
        kernel2.setArg<cl::Buffer>(4, cl.getVelm().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(5, integration.getPosDelta().getDeviceBuffer());
7515
        selectSizeKernel.setArg<cl_int>(0, numAtoms);
7516
        selectSizeKernel.setArg<cl::Buffer>(3, cl.getIntegrationUtilities().getStepSize().getDeviceBuffer());
7517
7518
        selectSizeKernel.setArg<cl::Buffer>(4, cl.getVelm().getDeviceBuffer());
        selectSizeKernel.setArg<cl::Buffer>(5, cl.getForce().getDeviceBuffer());
7519
7520
        int elementSize = (useDouble ? sizeof(cl_double) : sizeof(cl_float));
        selectSizeKernel.setArg(6, blockSize*elementSize, NULL);
7521
7522
7523
7524
    }

    // Select the step size to use.

7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
    double maxStepSize = maxTime-cl.getTime();
    float maxStepSizeFloat = (float) maxStepSize;
    if (useDouble) {
        selectSizeKernel.setArg<cl_double>(1, maxStepSize);
        selectSizeKernel.setArg<cl_double>(2, integrator.getErrorTolerance());
    }
    else {
        selectSizeKernel.setArg<cl_float>(1, maxStepSizeFloat);
        selectSizeKernel.setArg<cl_float>(2, (cl_float) integrator.getErrorTolerance());
    }
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
    cl.executeKernel(selectSizeKernel, blockSize, blockSize);

    // Call the first integration kernel.

    cl.executeKernel(kernel1, numAtoms);

    // Apply constraints.

    integration.applyConstraints(integrator.getConstraintTolerance());

    // Call the second integration kernel.

    cl.executeKernel(kernel2, numAtoms);
7548
    integration.computeVirtualSites();
7549
7550
7551
7552
7553
7554
    
    // Reduce UI lag.
    
#ifdef WIN32
    cl.getQueue().flush();
#endif
7555
7556
7557

    // Update the time and step count.

7558
7559
    double dt = cl.getIntegrationUtilities().getLastStepSize();
    double time = cl.getTime()+dt;
7560
7561
7562
7563
7564
7565
7566
7567
    if (useDouble) {
        if (dt == maxStepSize)
            time = maxTime; // Avoid round-off error
    }
    else {
        if (dt == maxStepSizeFloat)
            time = maxTime; // Avoid round-off error
    }
7568
7569
    cl.setTime(time);
    cl.setStepCount(cl.getStepCount()+1);
7570
    cl.reorderAtoms();
7571
    return dt;
7572
7573
}

7574
7575
7576
7577
double OpenCLIntegrateVariableVerletStepKernel::computeKineticEnergy(ContextImpl& context, const VariableVerletIntegrator& integrator) {
    return cl.getIntegrationUtilities().computeKineticEnergy(0.5*integrator.getStepSize());
}

7578
void OpenCLIntegrateVariableLangevinStepKernel::initialize(const System& system, const VariableLangevinIntegrator& integrator) {
7579
    cl.getPlatformData().initializeContexts(system);
7580
7581
    cl.getIntegrationUtilities().initRandomNumberGenerator(integrator.getRandomNumberSeed());
    map<string, string> defines;
7582
7583
    defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
    defines["PADDED_NUM_ATOMS"] = cl.intToString(cl.getPaddedNumAtoms());
7584
    cl::Program program = cl.createProgram(OpenCLKernelSources::langevin, defines, "");
7585
7586
7587
    kernel1 = cl::Kernel(program, "integrateLangevinPart1");
    kernel2 = cl::Kernel(program, "integrateLangevinPart2");
    selectSizeKernel = cl::Kernel(program, "selectLangevinStepSize");
peastman's avatar
peastman committed
7588
    params.initialize(cl, 3, cl.getUseDoublePrecision() || cl.getUseMixedPrecision() ? sizeof(cl_double) : sizeof(cl_float), "langevinParams");
Peter Eastman's avatar
Peter Eastman committed
7589
    blockSize = min(256, system.getNumParticles());
peastman's avatar
peastman committed
7590
    blockSize = max(blockSize, params.getSize());
7591
    blockSize = min(blockSize, (int) selectSizeKernel.getWorkGroupInfo<CL_KERNEL_WORK_GROUP_SIZE>(cl.getDevice()));
7592
7593
}

7594
double OpenCLIntegrateVariableLangevinStepKernel::execute(ContextImpl& context, const VariableLangevinIntegrator& integrator, double maxTime) {
7595
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
7596
    int numAtoms = cl.getNumAtoms();
7597
    bool useDouble = cl.getUseDoublePrecision() || cl.getUseMixedPrecision();
7598
7599
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
7600
7601
7602
        kernel1.setArg<cl::Buffer>(0, cl.getVelm().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(1, cl.getForce().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(2, integration.getPosDelta().getDeviceBuffer());
peastman's avatar
peastman committed
7603
        kernel1.setArg<cl::Buffer>(3, params.getDeviceBuffer());
7604
7605
7606
        kernel1.setArg<cl::Buffer>(4, integration.getStepSize().getDeviceBuffer());
        kernel1.setArg<cl::Buffer>(5, integration.getRandom().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(0, cl.getPosq().getDeviceBuffer());
7607
7608
7609
7610
        setPosqCorrectionArg(cl, kernel2, 1);
        kernel2.setArg<cl::Buffer>(2, integration.getPosDelta().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(3, cl.getVelm().getDeviceBuffer());
        kernel2.setArg<cl::Buffer>(4, integration.getStepSize().getDeviceBuffer());
7611
        selectSizeKernel.setArg<cl::Buffer>(4, integration.getStepSize().getDeviceBuffer());
7612
7613
        selectSizeKernel.setArg<cl::Buffer>(5, cl.getVelm().getDeviceBuffer());
        selectSizeKernel.setArg<cl::Buffer>(6, cl.getForce().getDeviceBuffer());
peastman's avatar
peastman committed
7614
        selectSizeKernel.setArg<cl::Buffer>(7, params.getDeviceBuffer());
7615
        int elementSize = (useDouble ? sizeof(cl_double) : sizeof(cl_float));
peastman's avatar
peastman committed
7616
        selectSizeKernel.setArg(8, params.getSize()*elementSize, NULL);
7617
        selectSizeKernel.setArg(9, blockSize*elementSize, NULL);
7618
7619
7620
7621
    }

    // Select the step size to use.

7622
7623
7624
7625
7626
    double maxStepSize = maxTime-cl.getTime();
    float maxStepSizeFloat = (float) maxStepSize;
    if (useDouble) {
        selectSizeKernel.setArg<cl_double>(0, maxStepSize);
        selectSizeKernel.setArg<cl_double>(1, integrator.getErrorTolerance());
7627
        selectSizeKernel.setArg<cl_double>(2, integrator.getFriction());
7628
7629
7630
7631
7632
        selectSizeKernel.setArg<cl_double>(3, BOLTZ*integrator.getTemperature());
    }
    else {
        selectSizeKernel.setArg<cl_float>(0, maxStepSizeFloat);
        selectSizeKernel.setArg<cl_float>(1, (cl_float) integrator.getErrorTolerance());
7633
        selectSizeKernel.setArg<cl_float>(2, (cl_float) integrator.getFriction());
7634
7635
        selectSizeKernel.setArg<cl_float>(3, (cl_float) (BOLTZ*integrator.getTemperature()));
    }
7636
7637
7638
7639
    cl.executeKernel(selectSizeKernel, blockSize, blockSize);

    // Call the first integration kernel.

7640
    kernel1.setArg<cl_uint>(6, integration.prepareRandomNumbers(cl.getPaddedNumAtoms()));
7641
7642
7643
7644
7645
7646
7647
7648
7649
    cl.executeKernel(kernel1, numAtoms);

    // Apply constraints.

    integration.applyConstraints(integrator.getConstraintTolerance());

    // Call the second integration kernel.

    cl.executeKernel(kernel2, numAtoms);
7650
    integration.computeVirtualSites();
7651
7652
7653
7654
7655
7656
    
    // Reduce UI lag.
    
#ifdef WIN32
    cl.getQueue().flush();
#endif
7657
7658
7659

    // Update the time and step count.

7660
7661
    double dt = cl.getIntegrationUtilities().getLastStepSize();
    double time = cl.getTime()+dt;
7662
7663
7664
7665
7666
7667
7668
7669
    if (useDouble) {
        if (dt == maxStepSize)
            time = maxTime; // Avoid round-off error
    }
    else {
        if (dt == maxStepSizeFloat)
            time = maxTime; // Avoid round-off error
    }
7670
7671
    cl.setTime(time);
    cl.setStepCount(cl.getStepCount()+1);
7672
    cl.reorderAtoms();
7673
    return dt;
7674
7675
}

7676
7677
7678
7679
double OpenCLIntegrateVariableLangevinStepKernel::computeKineticEnergy(ContextImpl& context, const VariableLangevinIntegrator& integrator) {
    return cl.getIntegrationUtilities().computeKineticEnergy(0.5*integrator.getStepSize());
}

7680
7681
class OpenCLIntegrateCustomStepKernel::ReorderListener : public OpenCLContext::ReorderListener {
public:
7682
    ReorderListener(OpenCLContext& cl, vector<OpenCLArray>& perDofValues, vector<vector<mm_float4> >& localPerDofValuesFloat, vector<vector<mm_double4> >& localPerDofValuesDouble, vector<bool>& deviceValuesAreCurrent) :
7683
            cl(cl), perDofValues(perDofValues), localPerDofValuesFloat(localPerDofValuesFloat), localPerDofValuesDouble(localPerDofValuesDouble), deviceValuesAreCurrent(deviceValuesAreCurrent) {
7684
7685
7686
7687
7688
7689
7690
7691
        int numAtoms = cl.getNumAtoms();
        lastAtomOrder.resize(numAtoms);
        for (int i = 0; i < numAtoms; i++)
            lastAtomOrder[i] = cl.getAtomIndex()[i];
    }
    void execute() {
        // Reorder the per-DOF variables to reflect the new atom order.

7692
        if (perDofValues.size() == 0)
7693
            return;
7694
        int numAtoms = cl.getNumAtoms();
7695
        const vector<int>& order = cl.getAtomIndex();
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
        for (int index = 0; index < perDofValues.size(); index++) {
            if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
                if (deviceValuesAreCurrent[index])
                    perDofValues[index].download(localPerDofValuesDouble[index]);
                vector<mm_double4> swap(numAtoms);
                for (int i = 0; i < numAtoms; i++)
                    swap[lastAtomOrder[i]] = localPerDofValuesDouble[index][i];
                for (int i = 0; i < numAtoms; i++)
                    localPerDofValuesDouble[index][i] = swap[order[i]];
                perDofValues[index].upload(localPerDofValuesDouble[index]);
7706
            }
7707
7708
7709
7710
7711
7712
7713
7714
7715
            else {
                if (deviceValuesAreCurrent[index])
                    perDofValues[index].download(localPerDofValuesFloat[index]);
                vector<mm_float4> swap(numAtoms);
                for (int i = 0; i < numAtoms; i++)
                    swap[lastAtomOrder[i]] = localPerDofValuesFloat[index][i];
                for (int i = 0; i < numAtoms; i++)
                    localPerDofValuesFloat[index][i] = swap[order[i]];
                perDofValues[index].upload(localPerDofValuesFloat[index]);
7716
            }
7717
            deviceValuesAreCurrent[index] = true;
7718
        }
7719
7720
7721
7722
7723
        for (int i = 0; i < numAtoms; i++)
            lastAtomOrder[i] = order[i];
    }
private:
    OpenCLContext& cl;
7724
7725
7726
7727
    vector<OpenCLArray>& perDofValues;
    vector<vector<mm_float4> >& localPerDofValuesFloat;
    vector<vector<mm_double4> >& localPerDofValuesDouble;
    vector<bool>& deviceValuesAreCurrent;
Peter Eastman's avatar
Peter Eastman committed
7728
    vector<int> lastAtomOrder;
7729
7730
};

7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
class OpenCLIntegrateCustomStepKernel::DerivFunction : public CustomFunction {
public:
    DerivFunction(map<string, double>& energyParamDerivs, const string& param) : energyParamDerivs(energyParamDerivs), param(param) {
    }
    int getNumArguments() const {
        return 0;
    }
    double evaluate(const double* arguments) const {
        return energyParamDerivs[param];
    }
    double evaluateDerivative(const double* arguments, const int* derivOrder) const {
        return 0;
    }
    CustomFunction* clone() const {
        return new DerivFunction(energyParamDerivs, param);
    }
private:
    map<string, double>& energyParamDerivs;
    string param;
};

7752
7753
7754
7755
void OpenCLIntegrateCustomStepKernel::initialize(const System& system, const CustomIntegrator& integrator) {
    cl.getPlatformData().initializeContexts(system);
    cl.getIntegrationUtilities().initRandomNumberGenerator(integrator.getRandomNumberSeed());
    numGlobalVariables = integrator.getNumGlobalVariables();
7756
    int elementSize = (cl.getUseDoublePrecision() || cl.getUseMixedPrecision() ? sizeof(double) : sizeof(float));
7757
    sumBuffer.initialize(cl, system.getNumParticles(), elementSize, "sumBuffer");
peastman's avatar
peastman committed
7758
    summedValue.initialize(cl, 1, elementSize, "summedValue");
7759
7760
7761
7762
7763
7764
7765
7766
    perDofValues.resize(integrator.getNumPerDofVariables());
    localPerDofValuesFloat.resize(perDofValues.size());
    localPerDofValuesDouble.resize(perDofValues.size());
    for (int i = 0; i < perDofValues.size(); i++)
        perDofValues[i].initialize(cl, system.getNumParticles(), 4*elementSize, "perDofVariables");
    localValuesAreCurrent.resize(integrator.getNumPerDofVariables(), false);
    deviceValuesAreCurrent.resize(integrator.getNumPerDofVariables(), false);
    cl.addReorderListener(new ReorderListener(cl, perDofValues, localPerDofValuesFloat, localPerDofValuesDouble, deviceValuesAreCurrent));
7767
7768
7769
    SimTKOpenMMUtilities::setRandomNumberSeed(integrator.getRandomNumberSeed());
}

7770
string OpenCLIntegrateCustomStepKernel::createPerDofComputation(const string& variable, const Lepton::ParsedExpression& expr, CustomIntegrator& integrator,
7771
        const string& forceName, const string& energyName, vector<const TabulatedFunction*>& functions, vector<pair<string, string> >& functionNames) {
7772
    string tempType = (cl.getSupportsDoublePrecision() ? "double3" : "float3");
7773
    string convert = (cl.getSupportsDoublePrecision() ? "convert_double3" : "");
7774
    map<string, Lepton::ParsedExpression> expressions;
7775
    expressions[tempType+" tempResult = "] = expr;
7776
    map<string, string> variables;
7777
7778
7779
7780
7781
    variables["x"] = convert+"(position.xyz)";
    variables["v"] = convert+"(velocity.xyz)";
    variables[forceName] = convert+"(f.xyz)";
    variables["gaussian"] = convert+"(gaussian.xyz)";
    variables["uniform"] = convert+"(uniform.xyz)";
7782
7783
    variables["m"] = "mass";
    variables["dt"] = "stepSize";
7784
    if (energyName != "")
Peter Eastman's avatar
Peter Eastman committed
7785
        variables[energyName] = "energy";
7786
    for (int i = 0; i < integrator.getNumGlobalVariables(); i++)
7787
        variables[integrator.getGlobalVariableName(i)] = "globals["+cl.intToString(globalVariableIndex[i])+"]";
7788
    for (int i = 0; i < integrator.getNumPerDofVariables(); i++)
7789
        variables[integrator.getPerDofVariableName(i)] = convert+"(perDof"+cl.intToString(i)+")";
7790
    for (int i = 0; i < (int) parameterNames.size(); i++)
7791
        variables[parameterNames[i]] = "globals["+cl.intToString(parameterVariableIndex[i])+"]";
7792
7793
    vector<pair<ExpressionTreeNode, string> > variableNodes;
    findExpressionsForDerivs(expr.getRootNode(), variableNodes);
peastman's avatar
peastman committed
7794
7795
    for (auto& var : variables)
        variableNodes.push_back(make_pair(ExpressionTreeNode(new Operation::Variable(var.first)), var.second));
7796
    string result = cl.getExpressionUtilities().createExpressions(expressions, variableNodes, functions, functionNames, "temp", tempType);
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
    if (variable == "x")
        result += "position.x = tempResult.x; position.y = tempResult.y; position.z = tempResult.z;\n";
    else if (variable == "v")
        result += "velocity.x = tempResult.x; velocity.y = tempResult.y; velocity.z = tempResult.z;\n";
    else if (variable == "")
        result += "sum[index] = tempResult.x+tempResult.y+tempResult.z;\n";
    else {
        for (int i = 0; i < integrator.getNumPerDofVariables(); i++)
            if (variable == integrator.getPerDofVariableName(i)) {
                string varName = "perDof"+cl.intToString(i);
                result += varName+".x = tempResult.x; "+varName+".y = tempResult.y; "+varName+".z = tempResult.z;\n";
            }
    }
7810
    return result;
7811
7812
}

7813
void OpenCLIntegrateCustomStepKernel::prepareForComputation(ContextImpl& context, CustomIntegrator& integrator, bool& forcesAreValid) {
7814
7815
7816
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
    int numAtoms = cl.getNumAtoms();
    int numSteps = integrator.getNumComputations();
7817
    bool useDouble = cl.getUseDoublePrecision() || cl.getUseMixedPrecision();
7818
    string tempType = (cl.getSupportsDoublePrecision() ? "double3" : "float3");
7819
    string perDofType = (useDouble ? "double4" : "float4");
7820
7821
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
7822
7823
7824
7825
        
        // Initialize various data structures.
        
        const map<string, double>& params = context.getParameters();
peastman's avatar
peastman committed
7826
7827
        for (auto& param : params)
            parameterNames.push_back(param.first);
7828
        kernels.resize(integrator.getNumComputations());
7829
7830
        requiredGaussian.resize(integrator.getNumComputations(), 0);
        requiredUniform.resize(integrator.getNumComputations(), 0);
7831
7832
7833
7834
        needsGlobals.resize(numSteps, false);
        globalExpressions.resize(numSteps);
        stepType.resize(numSteps);
        stepTarget.resize(numSteps);
7835
        merged.resize(numSteps, false);
7836
        modifiesParameters = false;
7837
7838
7839
        sumWorkGroupSize = cl.getDevice().getInfo<CL_DEVICE_MAX_WORK_GROUP_SIZE>();
        if (sumWorkGroupSize > 512)
            sumWorkGroupSize = 512;
7840
        map<string, string> defines;
7841
        defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
7842
        defines["WORK_GROUP_SIZE"] = cl.intToString(sumWorkGroupSize);
7843
7844
7845
7846
7847
7848
7849

        // Record the tabulated functions.

        map<string, Lepton::CustomFunction*> functions;
        vector<pair<string, string> > functionNames;
        vector<const TabulatedFunction*> functionList;
        vector<string> tableTypes;
peastman's avatar
peastman committed
7850
        tabulatedFunctions.resize(integrator.getNumTabulatedFunctions());
7851
7852
7853
7854
7855
7856
7857
7858
        for (int i = 0; i < integrator.getNumTabulatedFunctions(); i++) {
            functionList.push_back(&integrator.getTabulatedFunction(i));
            string name = integrator.getTabulatedFunctionName(i);
            string arrayName = "table"+cl.intToString(i);
            functionNames.push_back(make_pair(name, arrayName));
            functions[name] = createReferenceTabulatedFunction(integrator.getTabulatedFunction(i));
            int width;
            vector<float> f = cl.getExpressionUtilities().computeFunctionCoefficients(integrator.getTabulatedFunction(i), width);
peastman's avatar
peastman committed
7859
7860
            tabulatedFunctions[i].initialize<float>(cl, f.size(), "TabulatedFunction");
            tabulatedFunctions[i].upload(f);
7861
7862
7863
7864
7865
7866
            if (width == 1)
                tableTypes.push_back("float");
            else
                tableTypes.push_back("float"+cl.intToString(width));
        }

7867
7868
7869
7870
7871
        // Record information about all the computation steps.

        vector<string> variable(numSteps);
        vector<int> forceGroup;
        vector<vector<Lepton::ParsedExpression> > expression;
7872
        CustomIntegratorUtilities::analyzeComputations(context, integrator, expression, comparisons, blockEnd, invalidatesForces, needsForces, needsEnergy, computeBothForceAndEnergy, forceGroup, functions);
7873
7874
7875
        for (int step = 0; step < numSteps; step++) {
            string expr;
            integrator.getComputationStep(step, stepType[step], variable[step], expr);
7876
            if (stepType[step] == CustomIntegrator::WhileBlockStart)
7877
                blockEnd[blockEnd[step]] = step; // Record where to branch back to.
7878
            if (stepType[step] == CustomIntegrator::ComputeGlobal || stepType[step] == CustomIntegrator::IfBlockStart || stepType[step] == CustomIntegrator::WhileBlockStart)
peastman's avatar
peastman committed
7879
7880
                for (auto& expr : expression[step])
                    globalExpressions[step].push_back(ParsedExpression(replaceDerivFunctions(expr.getRootNode(), context)).createCompiledExpression());
7881
7882
        }
        for (int step = 0; step < numSteps; step++) {
peastman's avatar
peastman committed
7883
7884
            for (auto& expr : globalExpressions[step])
                expressionSet.registerExpression(expr);
7885
7886
        }
        
7887
        // Record the indices for variables in the CompiledExpressionSet.
7888
        
7889
7890
7891
7892
7893
        gaussianVariableIndex = expressionSet.getVariableIndex("gaussian");
        uniformVariableIndex = expressionSet.getVariableIndex("uniform");
        dtVariableIndex = expressionSet.getVariableIndex("dt");
        for (int i = 0; i < integrator.getNumGlobalVariables(); i++)
            globalVariableIndex.push_back(expressionSet.getVariableIndex(integrator.getGlobalVariableName(i)));
peastman's avatar
peastman committed
7894
7895
        for (auto& name : parameterNames)
            parameterVariableIndex.push_back(expressionSet.getVariableIndex(name));
7896
7897
7898
7899

        // Record the variable names and flags for the force and energy in each step.

        forceGroupFlags.resize(numSteps, -1);
7900
        vector<string> forceGroupName;
7901
        vector<string> energyGroupName;
7902
        for (int i = 0; i < 32; i++) {
7903
7904
7905
7906
7907
7908
            stringstream fname;
            fname << "f" << i;
            forceGroupName.push_back(fname.str());
            stringstream ename;
            ename << "energy" << i;
            energyGroupName.push_back(ename.str());
7909
7910
        }
        vector<string> forceName(numSteps, "f");
7911
        vector<string> energyName(numSteps, "energy");
7912
        stepEnergyVariableIndex.resize(numSteps, expressionSet.getVariableIndex("energy"));
7913
        for (int step = 0; step < numSteps; step++) {
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
            if (needsForces[step] && forceGroup[step] > -1)
                forceName[step] = forceGroupName[forceGroup[step]];
            if (needsEnergy[step] && forceGroup[step] > -1) {
                energyName[step] = energyGroupName[forceGroup[step]];
                stepEnergyVariableIndex[step] = expressionSet.getVariableIndex(energyName[step]);
            }
            if (forceGroup[step] > -1)
                forceGroupFlags[step] = 1<<forceGroup[step];
            if (forceGroupFlags[step] == -2 && step > 0)
                forceGroupFlags[step] = forceGroupFlags[step-1];
peastman's avatar
peastman committed
7924
7925
7926
7927
            if (forceGroupFlags[step] != -2 && savedForces.find(forceGroupFlags[step]) == savedForces.end()) {
                savedForces[forceGroupFlags[step]] = OpenCLArray();
                savedForces[forceGroupFlags[step]].initialize(cl, cl.getForce().getSize(), cl.getForce().getElementSize(), "savedForces");
            }
7928
7929
7930
7931
7932
7933
7934
        }
        
        // Allocate space for storing global values, both on the host and the device.
        
        globalValuesFloat.resize(expressionSet.getNumVariables());
        globalValuesDouble.resize(expressionSet.getNumVariables());
        int elementSize = (cl.getUseDoublePrecision() || cl.getUseMixedPrecision() ? sizeof(double) : sizeof(float));
peastman's avatar
peastman committed
7935
        globalValues.initialize(cl, expressionSet.getNumVariables(), elementSize, "globalValues");
7936
7937
7938
7939
7940
7941
7942
7943
7944
        for (int i = 0; i < integrator.getNumGlobalVariables(); i++) {
            globalValuesDouble[globalVariableIndex[i]] = initialGlobalVariables[i];
            expressionSet.setVariable(globalVariableIndex[i], initialGlobalVariables[i]);
        }
        for (int i = 0; i < (int) parameterVariableIndex.size(); i++) {
            double value = context.getParameter(parameterNames[i]);
            globalValuesDouble[parameterVariableIndex[i]] = value;
            expressionSet.setVariable(parameterVariableIndex[i], value);
        }
7945
7946
7947
        int numContextParams = context.getParameters().size();
        localPerDofEnergyParamDerivsFloat.resize(numContextParams);
        localPerDofEnergyParamDerivsDouble.resize(numContextParams);
peastman's avatar
peastman committed
7948
        perDofEnergyParamDerivs.initialize(cl, max(1, numContextParams), elementSize, "perDofEnergyParamDerivs");
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
        
        // Record information about the targets of steps that will be stored in global variables.
        
        for (int step = 0; step < numSteps; step++) {
            if (stepType[step] == CustomIntegrator::ComputeGlobal || stepType[step] == CustomIntegrator::ComputeSum) {
                if (variable[step] == "dt")
                    stepTarget[step].type = DT;
                for (int i = 0; i < integrator.getNumGlobalVariables(); i++)
                    if (variable[step] == integrator.getGlobalVariableName(i))
                        stepTarget[step].type = VARIABLE;
peastman's avatar
peastman committed
7959
7960
                for (auto& name : parameterNames)
                    if (variable[step] == name) {
7961
7962
                        stepTarget[step].type = PARAMETER;
                        modifiesParameters = true;
7963
                    }
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
                stepTarget[step].variableIndex = expressionSet.getVariableIndex(variable[step]);
            }
        }

        // Identify which per-DOF steps are going to require global variables or context parameters.

        for (int step = 0; step < numSteps; step++) {
            if (stepType[step] == CustomIntegrator::ComputePerDof || stepType[step] == CustomIntegrator::ComputeSum) {
                for (int i = 0; i < integrator.getNumGlobalVariables(); i++)
                    if (usesVariable(expression[step][0], integrator.getGlobalVariableName(i)))
                        needsGlobals[step] = true;
peastman's avatar
peastman committed
7975
7976
                for (auto& name : parameterNames)
                    if (usesVariable(expression[step][0], name))
7977
                        needsGlobals[step] = true;
7978
            }
7979
7980
7981
7982
        }
        
        // Determine how each step will represent the position (as just a value, or a value plus a delta).
        
peastman's avatar
peastman committed
7983
        hasAnyConstraints = (context.getSystem().getNumConstraints() > 0);
7984
7985
        vector<bool> storePosAsDelta(numSteps, false);
        vector<bool> loadPosAsDelta(numSteps, false);
peastman's avatar
peastman committed
7986
7987
7988
7989
7990
        if (hasAnyConstraints) {
            bool beforeConstrain = false;
            for (int step = numSteps-1; step >= 0; step--) {
                if (stepType[step] == CustomIntegrator::ConstrainPositions)
                    beforeConstrain = true;
peastman's avatar
peastman committed
7991
                else if (stepType[step] == CustomIntegrator::ComputePerDof && variable[step] == "x" && beforeConstrain) {
peastman's avatar
peastman committed
7992
                    storePosAsDelta[step] = true;
peastman's avatar
peastman committed
7993
7994
                    beforeConstrain = false;
                }
peastman's avatar
peastman committed
7995
7996
7997
7998
7999
8000
8001
8002
8003
            }
            bool storedAsDelta = false;
            for (int step = 0; step < numSteps; step++) {
                loadPosAsDelta[step] = storedAsDelta;
                if (storePosAsDelta[step] == true)
                    storedAsDelta = true;
                if (stepType[step] == CustomIntegrator::ConstrainPositions)
                    storedAsDelta = false;
            }
8004
8005
        }
        
8006
8007
8008
        // Identify steps that can be merged into a single kernel.
        
        for (int step = 1; step < numSteps; step++) {
8009
            if (invalidatesForces[step-1] || forceGroupFlags[step] != forceGroupFlags[step-1])
8010
                continue;
8011
            if (stepType[step-1] == CustomIntegrator::ComputePerDof && stepType[step] == CustomIntegrator::ComputePerDof)
8012
8013
                merged[step] = true;
        }
8014
8015
8016
8017
8018
        for (int step = numSteps-1; step > 0; step--)
            if (merged[step]) {
                needsForces[step-1] = (needsForces[step] || needsForces[step-1]);
                needsEnergy[step-1] = (needsEnergy[step] || needsEnergy[step-1]);
                needsGlobals[step-1] = (needsGlobals[step] || needsGlobals[step-1]);
Peter Eastman's avatar
Peter Eastman committed
8019
                computeBothForceAndEnergy[step-1] = (computeBothForceAndEnergy[step] || computeBothForceAndEnergy[step-1]);
8020
            }
8021
        
8022
8023
8024
        // Loop over all steps and create the kernels for them.
        
        for (int step = 0; step < numSteps; step++) {
8025
            if ((stepType[step] == CustomIntegrator::ComputePerDof || stepType[step] == CustomIntegrator::ComputeSum) && !merged[step]) {
8026
8027
8028
                // Compute a per-DOF value.
                
                stringstream compute;
8029
                for (int i = 0; i < perDofValues.size(); i++)
8030
                    compute << tempType<<" perDof"<<cl.intToString(i)<<" = convert_"<<tempType<<"(perDofValues"<<cl.intToString(i)<<"[index].xyz);\n";
8031
                int numGaussian = 0, numUniform = 0;
8032
                for (int j = step; j < numSteps && (j == step || merged[j]); j++) {
8033
8034
                    numGaussian += numAtoms*usesVariable(expression[j][0], "gaussian");
                    numUniform += numAtoms*usesVariable(expression[j][0], "uniform");
8035
                    compute << "{\n";
8036
                    if (numGaussian > 0)
8037
                        compute << "float4 gaussian = gaussianValues[gaussianIndex+index];\n";
8038
                    if (numUniform > 0)
8039
                        compute << "float4 uniform = uniformValues[uniformIndex+index];\n";
8040
                    compute << createPerDofComputation(stepType[j] == CustomIntegrator::ComputePerDof ? variable[j] : "", expression[j][0], integrator, forceName[j], energyName[j], functionList, functionNames);
8041
8042
8043
                    if (variable[j] == "x") {
                        if (storePosAsDelta[j]) {
                            if (cl.getSupportsDoublePrecision())
8044
                                compute << "posDelta[index] = convert_mixed4(convert_double4(position)-convert_double4(loadPos(posq, posqCorrection, index)));\n";
8045
8046
8047
                            else
                                compute << "posDelta[index] = position-posq[index];\n";
                        }
8048
                        else
8049
                            compute << "storePos(posq, posqCorrection, index, position);\n";
8050
                    }
8051
                    else if (variable[j] == "v")
8052
                        compute << "velm[index] = convert_mixed4(velocity);\n";
8053
                    else {
8054
                        for (int i = 0; i < perDofValues.size(); i++)
8055
                            compute << "perDofValues"<<cl.intToString(i)<<"[index] = ("<<perDofType<<") (perDof"<<cl.intToString(i)<<".x, perDof"<<cl.intToString(i)<<".y, perDof"<<cl.intToString(i)<<".z, 0);\n";
8056
                    }
8057
                    if (numGaussian > 0)
8058
                        compute << "gaussianIndex += NUM_ATOMS;\n";
8059
                    if (numUniform > 0)
8060
                        compute << "uniformIndex += NUM_ATOMS;\n";
8061
                    compute << "}\n";
8062
8063
8064
8065
                }
                map<string, string> replacements;
                replacements["COMPUTE_STEP"] = compute.str();
                stringstream args;
8066
8067
8068
                for (int i = 0; i < perDofValues.size(); i++) {
                    string valueName = "perDofValues"+cl.intToString(i);
                    args << ", __global " << perDofType << "* restrict " << valueName;
8069
                }
8070
8071
                for (int i = 0; i < (int) tableTypes.size(); i++)
                    args << ", __global const " << tableTypes[i]<< "* restrict table" << i;
8072
                replacements["PARAMETER_ARGUMENTS"] = args.str();
8073
8074
8075
8076
                if (loadPosAsDelta[step])
                    defines["LOAD_POS_AS_DELTA"] = "1";
                else if (defines.find("LOAD_POS_AS_DELTA") != defines.end())
                    defines.erase("LOAD_POS_AS_DELTA");
8077
8078
8079
                cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customIntegratorPerDof, replacements), defines);
                cl::Kernel kernel = cl::Kernel(program, "computePerDof");
                kernels[step].push_back(kernel);
8080
8081
                requiredGaussian[step] = numGaussian;
                requiredUniform[step] = numUniform;
8082
8083
                int index = 0;
                kernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
8084
                setPosqCorrectionArg(cl, kernel, index++);
8085
8086
8087
8088
                kernel.setArg<cl::Buffer>(index++, integration.getPosDelta().getDeviceBuffer());
                kernel.setArg<cl::Buffer>(index++, cl.getVelm().getDeviceBuffer());
                kernel.setArg<cl::Buffer>(index++, cl.getForce().getDeviceBuffer());
                kernel.setArg<cl::Buffer>(index++, integration.getStepSize().getDeviceBuffer());
peastman's avatar
peastman committed
8089
8090
                kernel.setArg<cl::Buffer>(index++, globalValues.getDeviceBuffer());
                kernel.setArg<cl::Buffer>(index++, sumBuffer.getDeviceBuffer());
Peter Eastman's avatar
Peter Eastman committed
8091
                index += 4;
peastman's avatar
peastman committed
8092
                kernel.setArg<cl::Buffer>(index++, perDofEnergyParamDerivs.getDeviceBuffer());
8093
8094
                for (auto& array : perDofValues)
                    kernel.setArg<cl::Memory>(index++, array.getDeviceBuffer());
peastman's avatar
peastman committed
8095
8096
                for (auto& array : tabulatedFunctions)
                    kernel.setArg<cl::Buffer>(index++, array.getDeviceBuffer());
8097
                if (stepType[step] == CustomIntegrator::ComputeSum) {
8098
8099
                    // Create a second kernel for this step that sums the values.

8100
                    program = cl.createProgram(OpenCLKernelSources::customIntegrator, defines);
8101
                    kernel = cl::Kernel(program, useDouble ? "computeDoubleSum" : "computeFloatSum");
8102
8103
                    kernels[step].push_back(kernel);
                    index = 0;
peastman's avatar
peastman committed
8104
8105
                    kernel.setArg<cl::Buffer>(index++, sumBuffer.getDeviceBuffer());
                    kernel.setArg<cl::Buffer>(index++, summedValue.getDeviceBuffer());
peastman's avatar
peastman committed
8106
                    kernel.setArg<cl_int>(index++, numAtoms);
8107
                }
8108
            }
8109
8110
8111
8112
8113
8114
8115
8116
            else if (stepType[step] == CustomIntegrator::ConstrainPositions) {
                // Apply position constraints.

                cl::Program program = cl.createProgram(OpenCLKernelSources::customIntegrator, defines);
                cl::Kernel kernel = cl::Kernel(program, "applyPositionDeltas");
                kernels[step].push_back(kernel);
                int index = 0;
                kernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
8117
                setPosqCorrectionArg(cl, kernel, index++);
8118
8119
                kernel.setArg<cl::Buffer>(index++, integration.getPosDelta().getDeviceBuffer());
            }
8120
        }
8121
        
8122
8123
8124
        // Initialize the random number generator.
        
        int maxUniformRandoms = 1;
peastman's avatar
peastman committed
8125
8126
        for (int required : requiredUniform)
            maxUniformRandoms = max(maxUniformRandoms, required);
peastman's avatar
peastman committed
8127
8128
8129
        uniformRandoms.initialize<mm_float4>(cl, maxUniformRandoms, "uniformRandoms");
        randomSeed.initialize<mm_int4>(cl, cl.getNumThreadBlocks()*OpenCLContext::ThreadBlockSize, "randomSeed");
        vector<mm_int4> seed(randomSeed.getSize());
8130
        int rseed = integrator.getRandomNumberSeed();
8131
        // A random seed of 0 means use a unique one
8132
8133
8134
        if (rseed == 0)
            rseed = osrngseed();
        unsigned int r = (unsigned int) (rseed+1);
peastman's avatar
peastman committed
8135
8136
8137
8138
8139
        for (auto& s : seed) {
            s.x = r = (1664525*r + 1013904223) & 0xFFFFFFFF;
            s.y = r = (1664525*r + 1013904223) & 0xFFFFFFFF;
            s.z = r = (1664525*r + 1013904223) & 0xFFFFFFFF;
            s.w = r = (1664525*r + 1013904223) & 0xFFFFFFFF;
8140
        }
peastman's avatar
peastman committed
8141
        randomSeed.upload(seed);
8142
8143
        cl::Program randomProgram = cl.createProgram(OpenCLKernelSources::customIntegrator, defines);
        randomKernel = cl::Kernel(randomProgram, "generateRandomNumbers");
8144
        randomKernel.setArg<cl_int>(0, maxUniformRandoms);
peastman's avatar
peastman committed
8145
8146
        randomKernel.setArg<cl::Buffer>(1, uniformRandoms.getDeviceBuffer());
        randomKernel.setArg<cl::Buffer>(2, randomSeed.getDeviceBuffer());
8147
        
8148
8149
8150
        // Create the kernel for computing kinetic energy.

        stringstream computeKE;
8151
        for (int i = 0; i < perDofValues.size(); i++)
8152
            computeKE << tempType<<" perDof"<<cl.intToString(i)<<" = convert_"<<tempType<<"(perDofValues"<<cl.intToString(i)<<"[index].xyz);\n";
8153
        Lepton::ParsedExpression keExpression = Lepton::Parser::parse(integrator.getKineticEnergyExpression()).optimize();
8154
        computeKE << createPerDofComputation("", keExpression, integrator, "f", "", functionList, functionNames);
8155
8156
8157
        map<string, string> replacements;
        replacements["COMPUTE_STEP"] = computeKE.str();
        stringstream args;
8158
8159
8160
        for (int i = 0; i < perDofValues.size(); i++) {
            string valueName = "perDofValues"+cl.intToString(i);
            args << ", __global " << perDofType << "* restrict " << valueName;
8161
        }
8162
8163
        for (int i = 0; i < (int) tableTypes.size(); i++)
            args << ", __global const " << tableTypes[i]<< "* restrict table" << i;
8164
8165
8166
        replacements["PARAMETER_ARGUMENTS"] = args.str();
        if (defines.find("LOAD_POS_AS_DELTA") != defines.end())
            defines.erase("LOAD_POS_AS_DELTA");
Peter Eastman's avatar
Peter Eastman committed
8167
        cl::Program program = cl.createProgram(cl.replaceStrings(OpenCLKernelSources::customIntegratorPerDof, replacements), defines);
8168
        kineticEnergyKernel = cl::Kernel(program, "computePerDof");
Peter Eastman's avatar
Peter Eastman committed
8169
        int index = 0;
8170
8171
8172
8173
8174
8175
        kineticEnergyKernel.setArg<cl::Buffer>(index++, cl.getPosq().getDeviceBuffer());
        setPosqCorrectionArg(cl, kineticEnergyKernel, index++);
        kineticEnergyKernel.setArg<cl::Buffer>(index++, integration.getPosDelta().getDeviceBuffer());
        kineticEnergyKernel.setArg<cl::Buffer>(index++, cl.getVelm().getDeviceBuffer());
        kineticEnergyKernel.setArg<cl::Buffer>(index++, cl.getForce().getDeviceBuffer());
        kineticEnergyKernel.setArg<cl::Buffer>(index++, integration.getStepSize().getDeviceBuffer());
peastman's avatar
peastman committed
8176
8177
        kineticEnergyKernel.setArg<cl::Buffer>(index++, globalValues.getDeviceBuffer());
        kineticEnergyKernel.setArg<cl::Buffer>(index++, sumBuffer.getDeviceBuffer());
8178
        index += 2;
peastman's avatar
peastman committed
8179
        kineticEnergyKernel.setArg<cl::Buffer>(index++, uniformRandoms.getDeviceBuffer());
Peter Eastman's avatar
Peter Eastman committed
8180
        if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision())
Peter Eastman's avatar
Peter Eastman committed
8181
8182
8183
            kineticEnergyKernel.setArg<cl_double>(index++, 0.0);
        else
            kineticEnergyKernel.setArg<cl_float>(index++, 0.0f);
peastman's avatar
peastman committed
8184
        kineticEnergyKernel.setArg<cl::Buffer>(index++, perDofEnergyParamDerivs.getDeviceBuffer());
8185
        for (auto& array : perDofValues)
8186
            kineticEnergyKernel.setArg<cl::Buffer>(index++, array.getDeviceBuffer());
peastman's avatar
peastman committed
8187
8188
        for (auto& array : tabulatedFunctions)
            kineticEnergyKernel.setArg<cl::Buffer>(index++, array.getDeviceBuffer());
8189
8190
8191
8192
8193
8194
8195
        keNeedsForce = usesVariable(keExpression, "f");

        // Create a second kernel to sum the values.

        program = cl.createProgram(OpenCLKernelSources::customIntegrator, defines);
        sumKineticEnergyKernel = cl::Kernel(program, useDouble ? "computeDoubleSum" : "computeFloatSum");
        index = 0;
peastman's avatar
peastman committed
8196
8197
        sumKineticEnergyKernel.setArg<cl::Buffer>(index++, sumBuffer.getDeviceBuffer());
        sumKineticEnergyKernel.setArg<cl::Buffer>(index++, summedValue.getDeviceBuffer());
peastman's avatar
peastman committed
8198
        sumKineticEnergyKernel.setArg<cl_int>(index++, numAtoms);
8199
8200
8201
8202
8203

        // Delete the custom functions.

        for (auto& function : functions)
            delete function.second;
8204
    }
8205

8206
    // Make sure all values (variables, parameters, etc.) are up to date.
8207
    
8208
8209
8210
8211
8212
8213
8214
8215
8216
    for (int i = 0; i < perDofValues.size(); i++) {
        if (!deviceValuesAreCurrent[i]) {
            if (useDouble)
                perDofValues[i].upload(localPerDofValuesDouble[i]);
            else
                perDofValues[i].upload(localPerDofValuesFloat[i]);
            deviceValuesAreCurrent[i] = true;
        }
        localValuesAreCurrent[i] = false;
8217
8218
    }
    double stepSize = integrator.getStepSize();
8219
    recordGlobalValue(stepSize, GlobalTarget(DT, dtVariableIndex), integrator);
8220
8221
8222
8223
8224
    for (int i = 0; i < (int) parameterNames.size(); i++) {
        double value = context.getParameter(parameterNames[i]);
        if (value != globalValuesDouble[parameterVariableIndex[i]]) {
            globalValuesDouble[parameterVariableIndex[i]] = value;
            deviceGlobalsAreCurrent = false;
8225
8226
        }
    }
8227
}
8228

8229
8230
8231
8232
8233
8234
8235
8236
8237
8238
8239
8240
8241
8242
ExpressionTreeNode OpenCLIntegrateCustomStepKernel::replaceDerivFunctions(const ExpressionTreeNode& node, ContextImpl& context) {
    // This is called recursively to identify calls to the deriv() function inside global expressions,
    // and replace them with a custom function that returns the correct value.
    
    const Operation& op = node.getOperation();
    if (op.getId() == Operation::CUSTOM && op.getName() == "deriv") {
        string param = node.getChildren()[1].getOperation().getName();
        if (context.getParameters().find(param) == context.getParameters().end())
            throw OpenMMException("The second argument to deriv() must be a context parameter");
        needsEnergyParamDerivs = true;
        return ExpressionTreeNode(new Operation::Custom("deriv", new DerivFunction(energyParamDerivs, param)));
    }
    else {
        vector<ExpressionTreeNode> children;
peastman's avatar
peastman committed
8243
8244
        for (auto& child : node.getChildren())
            children.push_back(replaceDerivFunctions(child, context));
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257
8258
8259
8260
8261
8262
8263
8264
        return ExpressionTreeNode(op.clone(), children);
    }
}

void OpenCLIntegrateCustomStepKernel::findExpressionsForDerivs(const ExpressionTreeNode& node, vector<pair<ExpressionTreeNode, string> >& variableNodes) {
    // This is called recursively to identify calls to the deriv() function inside per-DOF expressions,
    // and record the code to replace them with.
    
    const Operation& op = node.getOperation();
    if (op.getId() == Operation::CUSTOM && op.getName() == "deriv") {
        string param = node.getChildren()[1].getOperation().getName();
        int index;
        for (index = 0; index < perDofEnergyParamDerivNames.size() && param != perDofEnergyParamDerivNames[index]; index++)
            ;
        if (index == perDofEnergyParamDerivNames.size())
            perDofEnergyParamDerivNames.push_back(param);
        variableNodes.push_back(make_pair(node, "energyParamDerivs["+cl.intToString(index)+"]"));
        needsEnergyParamDerivs = true;
    }
    else {
peastman's avatar
peastman committed
8265
8266
        for (auto& child : node.getChildren())
            findExpressionsForDerivs(child, variableNodes);
8267
8268
8269
    }
}

8270
8271
8272
8273
8274
void OpenCLIntegrateCustomStepKernel::execute(ContextImpl& context, CustomIntegrator& integrator, bool& forcesAreValid) {
    prepareForComputation(context, integrator, forcesAreValid);
    OpenCLIntegrationUtilities& integration = cl.getIntegrationUtilities();
    int numAtoms = cl.getNumAtoms();
    int numSteps = integrator.getNumComputations();
8275
8276
    if (!forcesAreValid)
        savedEnergy.clear();
8277
    
8278
8279
    // Loop over computation steps in the integrator and execute them.

8280
8281
    for (int step = 0; step < numSteps; ) {
        int nextStep = step+1;
8282
        int forceGroups = forceGroupFlags[step];
8283
        int lastForceGroups = context.getLastForceGroups();
8284
8285
8286
        bool haveForces = (!needsForces[step] || (forcesAreValid && lastForceGroups == forceGroups));
        bool haveEnergy = (!needsEnergy[step] || savedEnergy.find(forceGroups) != savedEnergy.end());
        if (!haveForces || !haveEnergy) {
Peter Eastman's avatar
Peter Eastman committed
8287
8288
8289
8290
8291
            if (forcesAreValid) {
                if (savedForces.find(lastForceGroups) != savedForces.end() && validSavedForces.find(lastForceGroups) == validSavedForces.end()) {
                    // The forces are still valid.  We just need a different force group right now.  Save the old
                    // forces in case we need them again.

peastman's avatar
peastman committed
8292
                    cl.getForce().copyTo(savedForces[lastForceGroups]);
Peter Eastman's avatar
Peter Eastman committed
8293
8294
                    validSavedForces.insert(lastForceGroups);
                }
8295
8296
8297
8298
            }
            else
                validSavedForces.clear();
            
8299
8300
8301
            // Recompute forces and/or energy.  Figure out what is actually needed
            // between now and the next time they get invalidated again.
            
8302
8303
            bool computeForce = (needsForces[step] || computeBothForceAndEnergy[step]);
            bool computeEnergy = (needsEnergy[step] || computeBothForceAndEnergy[step]);
8304
            if (!computeEnergy && validSavedForces.find(forceGroups) != validSavedForces.end()) {
8305
8306
                // We can just restore the forces we saved earlier.
                
peastman's avatar
peastman committed
8307
                savedForces[forceGroups].copyTo(cl.getForce());
8308
                context.getLastForceGroups() = forceGroups;
8309
8310
8311
            }
            else {
                recordChangedParameters(context);
8312
8313
                energy = context.calcForcesAndEnergy(computeForce, computeEnergy, forceGroups);
                savedEnergy[forceGroups] = energy;
8314
8315
8316
8317
8318
8319
                if (needsEnergyParamDerivs) {
                    context.getEnergyParameterDerivatives(energyParamDerivs);
                    if (perDofEnergyParamDerivNames.size() > 0) {
                        if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
                            for (int i = 0; i < perDofEnergyParamDerivNames.size(); i++)
                                localPerDofEnergyParamDerivsDouble[i] = energyParamDerivs[perDofEnergyParamDerivNames[i]];
peastman's avatar
peastman committed
8320
                            perDofEnergyParamDerivs.upload(localPerDofEnergyParamDerivsDouble);
8321
8322
8323
8324
                        }
                        else {
                            for (int i = 0; i < perDofEnergyParamDerivNames.size(); i++)
                                localPerDofEnergyParamDerivsFloat[i] = (float) energyParamDerivs[perDofEnergyParamDerivNames[i]];
peastman's avatar
peastman committed
8325
                            perDofEnergyParamDerivs.upload(localPerDofEnergyParamDerivsFloat);
8326
8327
8328
8329
                        }
                    }
                }
                forcesAreValid = true;
8330
            }
8331
        }
8332
8333
        if (needsEnergy[step])
            energy = savedEnergy[forceGroups];
8334
8335
8336
8337
        if (needsGlobals[step] && !deviceGlobalsAreCurrent) {
            // Upload the global values to the device.
            
            if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision())
peastman's avatar
peastman committed
8338
                globalValues.upload(globalValuesDouble);
8339
8340
8341
            else {
                for (int j = 0; j < (int) globalValuesDouble.size(); j++)
                    globalValuesFloat[j] = (float) globalValuesDouble[j];
peastman's avatar
peastman committed
8342
                globalValues.upload(globalValuesFloat);
8343
            }
8344
        }
8345
        bool stepInvalidatesForces = invalidatesForces[step];
8346
8347
8348
        if (stepType[step] == CustomIntegrator::ComputePerDof && !merged[step]) {
            kernels[step][0].setArg<cl_uint>(9, integration.prepareRandomNumbers(requiredGaussian[step]));
            kernels[step][0].setArg<cl::Buffer>(8, integration.getRandom().getDeviceBuffer());
peastman's avatar
peastman committed
8349
            kernels[step][0].setArg<cl::Buffer>(10, uniformRandoms.getDeviceBuffer());
Peter Eastman's avatar
Peter Eastman committed
8350
            if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision())
8351
                kernels[step][0].setArg<cl_double>(11, energy);
Peter Eastman's avatar
Peter Eastman committed
8352
            else
8353
8354
8355
                kernels[step][0].setArg<cl_float>(11, (cl_float) energy);
            if (requiredUniform[step] > 0)
                cl.executeKernel(randomKernel, numAtoms);
peastman's avatar
peastman committed
8356
            cl.executeKernel(kernels[step][0], numAtoms, 128);
8357
8358
8359
8360
8361
        }
        else if (stepType[step] == CustomIntegrator::ComputeGlobal) {
            expressionSet.setVariable(uniformVariableIndex, SimTKOpenMMUtilities::getUniformlyDistributedRandomNumber());
            expressionSet.setVariable(gaussianVariableIndex, SimTKOpenMMUtilities::getNormallyDistributedRandomNumber());
            expressionSet.setVariable(stepEnergyVariableIndex[step], energy);
8362
            recordGlobalValue(globalExpressions[step][0].evaluate(), stepTarget[step], integrator);
8363
8364
8365
8366
        }
        else if (stepType[step] == CustomIntegrator::ComputeSum) {
            kernels[step][0].setArg<cl_uint>(9, integration.prepareRandomNumbers(requiredGaussian[step]));
            kernels[step][0].setArg<cl::Buffer>(8, integration.getRandom().getDeviceBuffer());
peastman's avatar
peastman committed
8367
            kernels[step][0].setArg<cl::Buffer>(10, uniformRandoms.getDeviceBuffer());
Peter Eastman's avatar
Peter Eastman committed
8368
            if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision())
8369
                kernels[step][0].setArg<cl_double>(11, energy);
Peter Eastman's avatar
Peter Eastman committed
8370
            else
8371
8372
                kernels[step][0].setArg<cl_float>(11, (cl_float) energy);
            if (requiredUniform[step] > 0)
8373
                cl.executeKernel(randomKernel, numAtoms);
peastman's avatar
peastman committed
8374
            cl.clearBuffer(sumBuffer);
peastman's avatar
peastman committed
8375
            cl.executeKernel(kernels[step][0], numAtoms, 128);
8376
            cl.executeKernel(kernels[step][1], sumWorkGroupSize, sumWorkGroupSize);
8377
8378
            if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
                double value;
peastman's avatar
peastman committed
8379
                summedValue.download(&value);
8380
                recordGlobalValue(value, stepTarget[step], integrator);
8381
8382
8383
            }
            else {
                float value;
peastman's avatar
peastman committed
8384
                summedValue.download(&value);
8385
                recordGlobalValue(value, stepTarget[step], integrator);
8386
            }
8387
        }
8388
        else if (stepType[step] == CustomIntegrator::UpdateContextState) {
8389
            recordChangedParameters(context);
8390
            stepInvalidatesForces = context.updateContextState();
8391
        }
8392
        else if (stepType[step] == CustomIntegrator::ConstrainPositions) {
peastman's avatar
peastman committed
8393
8394
8395
8396
            if (hasAnyConstraints) {
                cl.getIntegrationUtilities().applyConstraints(integrator.getConstraintTolerance());
                cl.executeKernel(kernels[step][0], numAtoms);
            }
8397
            cl.getIntegrationUtilities().computeVirtualSites();
8398
        }
8399
        else if (stepType[step] == CustomIntegrator::ConstrainVelocities) {
8400
8401
            cl.getIntegrationUtilities().applyVelocityConstraints(integrator.getConstraintTolerance());
        }
8402
        else if (stepType[step] == CustomIntegrator::IfBlockStart) {
8403
8404
8405
            if (!evaluateCondition(step))
                nextStep = blockEnd[step]+1;
        }
8406
        else if (stepType[step] == CustomIntegrator::WhileBlockStart) {
8407
8408
8409
            if (!evaluateCondition(step))
                nextStep = blockEnd[step]+1;
        }
8410
        else if (stepType[step] == CustomIntegrator::BlockEnd) {
8411
8412
8413
            if (blockEnd[step] != -1)
                nextStep = blockEnd[step]; // Return to the start of a while block.
        }
8414
        if (stepInvalidatesForces) {
8415
            forcesAreValid = false;
8416
8417
            savedEnergy.clear();
        }
8418
        step = nextStep;
8419
    }
8420
    recordChangedParameters(context);
8421
8422
8423

    // Update the time and step count.

8424
    cl.setTime(cl.getTime()+integrator.getStepSize());
8425
    cl.setStepCount(cl.getStepCount()+1);
8426
    cl.reorderAtoms();
8427
8428
8429
8430
    if (cl.getAtomsWereReordered()) {
        forcesAreValid = false;
        validSavedForces.clear();
    }
8431
8432
8433
8434
8435
8436
    
    // Reduce UI lag.
    
#ifdef WIN32
    cl.getQueue().flush();
#endif
8437
8438
}

8439
8440
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
bool OpenCLIntegrateCustomStepKernel::evaluateCondition(int step) {
    expressionSet.setVariable(uniformVariableIndex, SimTKOpenMMUtilities::getUniformlyDistributedRandomNumber());
    expressionSet.setVariable(gaussianVariableIndex, SimTKOpenMMUtilities::getNormallyDistributedRandomNumber());
    expressionSet.setVariable(stepEnergyVariableIndex[step], energy);
    double lhs = globalExpressions[step][0].evaluate();
    double rhs = globalExpressions[step][1].evaluate();
    switch (comparisons[step]) {
        case CustomIntegratorUtilities::EQUAL:
            return (lhs == rhs);
        case CustomIntegratorUtilities::LESS_THAN:
            return (lhs < rhs);
        case CustomIntegratorUtilities::GREATER_THAN:
            return (lhs > rhs);
        case CustomIntegratorUtilities::NOT_EQUAL:
            return (lhs != rhs);
        case CustomIntegratorUtilities::LESS_THAN_OR_EQUAL:
            return (lhs <= rhs);
        case CustomIntegratorUtilities::GREATER_THAN_OR_EQUAL:
            return (lhs >= rhs);
    }
    throw OpenMMException("Invalid comparison operator");
}

8462
8463
8464
8465
8466
8467
8468
8469
8470
double OpenCLIntegrateCustomStepKernel::computeKineticEnergy(ContextImpl& context, CustomIntegrator& integrator, bool& forcesAreValid) {
    prepareForComputation(context, integrator, forcesAreValid);
    if (keNeedsForce && !forcesAreValid) {
        // Compute the force.  We want to then mark that forces are valid, which means also computing
        // potential energy if any steps will expect it to be valid too.
        
        bool willNeedEnergy = false;
        for (int i = 0; i < integrator.getNumComputations(); i++)
            willNeedEnergy |= needsEnergy[i];
Peter Eastman's avatar
Peter Eastman committed
8471
        energy = context.calcForcesAndEnergy(true, willNeedEnergy, -1);
8472
8473
        forcesAreValid = true;
    }
peastman's avatar
peastman committed
8474
    cl.clearBuffer(sumBuffer);
8475
8476
    kineticEnergyKernel.setArg<cl::Buffer>(8, cl.getIntegrationUtilities().getRandom().getDeviceBuffer());
    kineticEnergyKernel.setArg<cl_uint>(9, 0);
8477
    cl.executeKernel(kineticEnergyKernel, cl.getNumAtoms());
8478
    cl.executeKernel(sumKineticEnergyKernel, sumWorkGroupSize, sumWorkGroupSize);
8479
8480
    if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
        double ke;
peastman's avatar
peastman committed
8481
        summedValue.download(&ke);
8482
8483
8484
8485
        return ke;
    }
    else {
        float ke;
peastman's avatar
peastman committed
8486
        summedValue.download(&ke);
8487
8488
8489
8490
        return ke;
    }
}

8491
void OpenCLIntegrateCustomStepKernel::recordGlobalValue(double value, GlobalTarget target, CustomIntegrator& integrator) {
8492
8493
    switch (target.type) {
        case DT:
8494
8495
            if (value != globalValuesDouble[dtVariableIndex])
                deviceGlobalsAreCurrent = false;
8496
            expressionSet.setVariable(dtVariableIndex, value);
8497
            globalValuesDouble[dtVariableIndex] = value;
8498
            cl.getIntegrationUtilities().setNextStepSize(value);
8499
            integrator.setStepSize(value);
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
            break;
        case VARIABLE:
        case PARAMETER:
            expressionSet.setVariable(target.variableIndex, value);
            globalValuesDouble[target.variableIndex] = value;
            deviceGlobalsAreCurrent = false;
            break;
    }
}

8510
8511
8512
void OpenCLIntegrateCustomStepKernel::recordChangedParameters(ContextImpl& context) {
    if (!modifiesParameters)
        return;
8513
8514
8515
8516
    for (int i = 0; i < (int) parameterNames.size(); i++) {
        double value = context.getParameter(parameterNames[i]);
        if (value != globalValuesDouble[parameterVariableIndex[i]])
            context.setParameter(parameterNames[i], globalValuesDouble[parameterVariableIndex[i]]);
8517
8518
8519
    }
}

8520
void OpenCLIntegrateCustomStepKernel::getGlobalVariables(ContextImpl& context, vector<double>& values) const {
peastman's avatar
peastman committed
8521
    if (!globalValues.isInitialized()) {
8522
8523
8524
        // The data structures haven't been created yet, so just return the list of values that was given earlier.
        
        values = initialGlobalVariables;
peastman's avatar
peastman committed
8525
        return;
8526
    }
8527
8528
8529
    values.resize(numGlobalVariables);
    for (int i = 0; i < numGlobalVariables; i++)
        values[i] = globalValuesDouble[globalVariableIndex[i]];
8530
8531
8532
}

void OpenCLIntegrateCustomStepKernel::setGlobalVariables(ContextImpl& context, const vector<double>& values) {
8533
8534
    if (numGlobalVariables == 0)
        return;
peastman's avatar
peastman committed
8535
    if (!globalValues.isInitialized()) {
8536
8537
8538
8539
8540
8541
8542
8543
        // The data structures haven't been created yet, so just store the list of values.
        
        initialGlobalVariables = values;
        return;
    }
    for (int i = 0; i < numGlobalVariables; i++) {
        globalValuesDouble[globalVariableIndex[i]] = values[i];
        expressionSet.setVariable(globalVariableIndex[i], values[i]);
8544
    }
8545
    deviceGlobalsAreCurrent = false;
8546
8547
8548
}

void OpenCLIntegrateCustomStepKernel::getPerDofVariable(ContextImpl& context, int variable, vector<Vec3>& values) const {
8549
    values.resize(perDofValues[variable].getSize());
8550
8551
    const vector<int>& order = cl.getAtomIndex();
    if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
8552
8553
8554
8555
8556
8557
8558
8559
        if (!localValuesAreCurrent[variable]) {
            perDofValues[variable].download(localPerDofValuesDouble[variable]);
            localValuesAreCurrent[variable] = true;
        }
        for (int i = 0; i < (int) values.size(); i++) {
            values[order[i]][0] = localPerDofValuesDouble[variable][i].x;
            values[order[i]][1] = localPerDofValuesDouble[variable][i].y;
            values[order[i]][2] = localPerDofValuesDouble[variable][i].z;
8560
8561
8562
        }
    }
    else {
8563
8564
8565
8566
8567
8568
8569
8570
        if (!localValuesAreCurrent[variable]) {
            perDofValues[variable].download(localPerDofValuesFloat[variable]);
            localValuesAreCurrent[variable] = true;
        }
        for (int i = 0; i < (int) values.size(); i++) {
            values[order[i]][0] = localPerDofValuesFloat[variable][i].x;
            values[order[i]][1] = localPerDofValuesFloat[variable][i].y;
            values[order[i]][2] = localPerDofValuesFloat[variable][i].z;
8571
8572
        }
    }
8573
8574
8575
}

void OpenCLIntegrateCustomStepKernel::setPerDofVariable(ContextImpl& context, int variable, const vector<Vec3>& values) {
8576
    const vector<int>& order = cl.getAtomIndex();
8577
8578
    localValuesAreCurrent[variable] = true;
    deviceValuesAreCurrent[variable] = false;
8579
    if (cl.getUseDoublePrecision() || cl.getUseMixedPrecision()) {
8580
        localPerDofValuesDouble[variable].resize(values.size());
8581
        for (int i = 0; i < (int) values.size(); i++)
8582
            localPerDofValuesDouble[variable][i] = mm_double4(values[order[i]][0], values[order[i]][1], values[order[i]][2], 0);
8583
8584
    }
    else {
8585
        localPerDofValuesFloat[variable].resize(values.size());
8586
        for (int i = 0; i < (int) values.size(); i++)
8587
            localPerDofValuesFloat[variable][i] = mm_float4(values[order[i]][0], values[order[i]][1], values[order[i]][2], 0);
8588
8589
8590
    }
}

8591
8592
8593
void OpenCLApplyAndersenThermostatKernel::initialize(const System& system, const AndersenThermostat& thermostat) {
    randomSeed = thermostat.getRandomNumberSeed();
    map<string, string> defines;
8594
    defines["NUM_ATOMS"] = cl.intToString(cl.getNumAtoms());
8595
    cl::Program program = cl.createProgram(OpenCLKernelSources::andersenThermostat, defines);
8596
    kernel = cl::Kernel(program, "applyAndersenThermostat");
Peter Eastman's avatar
Peter Eastman committed
8597
    cl.getIntegrationUtilities().initRandomNumberGenerator(randomSeed);
8598
8599
8600
8601

    // Create the arrays with the group definitions.

    vector<vector<int> > groups = AndersenThermostatImpl::calcParticleGroups(system);
peastman's avatar
peastman committed
8602
8603
    atomGroups.initialize<int>(cl, cl.getNumAtoms(), "atomGroups");
    vector<int> atoms(atomGroups.getSize());
8604
8605
8606
8607
    for (int i = 0; i < (int) groups.size(); i++) {
        for (int j = 0; j < (int) groups[i].size(); j++)
            atoms[groups[i][j]] = i;
    }
peastman's avatar
peastman committed
8608
    atomGroups.upload(atoms);
8609
8610
8611
8612
8613
8614
8615
8616
}

void OpenCLApplyAndersenThermostatKernel::execute(ContextImpl& context) {
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;
        kernel.setArg<cl::Buffer>(2, cl.getVelm().getDeviceBuffer());
        kernel.setArg<cl::Buffer>(3, cl.getIntegrationUtilities().getStepSize().getDeviceBuffer());
        kernel.setArg<cl::Buffer>(4, cl.getIntegrationUtilities().getRandom().getDeviceBuffer());
peastman's avatar
peastman committed
8617
        kernel.setArg<cl::Buffer>(6, atomGroups.getDeviceBuffer());
8618
8619
8620
8621
8622
8623
8624
    }
    kernel.setArg<cl_float>(0, (cl_float) context.getParameter(AndersenThermostat::CollisionFrequency()));
    kernel.setArg<cl_float>(1, (cl_float) (BOLTZ*context.getParameter(AndersenThermostat::Temperature())));
    kernel.setArg<cl_uint>(5, cl.getIntegrationUtilities().prepareRandomNumbers(cl.getPaddedNumAtoms()));
    cl.executeKernel(kernel, cl.getNumAtoms());
}

8625
void OpenCLApplyMonteCarloBarostatKernel::initialize(const System& system, const Force& thermostat) {
peastman's avatar
peastman committed
8626
8627
    savedPositions.initialize(cl, cl.getPaddedNumAtoms(), cl.getUseDoublePrecision() ? sizeof(mm_double4) : sizeof(mm_float4), "savedPositions");
    savedForces.initialize(cl, cl.getPaddedNumAtoms(), cl.getUseDoublePrecision() ? sizeof(mm_double4) : sizeof(mm_float4), "savedForces");
8628
    cl::Program program = cl.createProgram(OpenCLKernelSources::monteCarloBarostat);
8629
    kernel = cl::Kernel(program, "scalePositions");
8630
8631
}

8632
void OpenCLApplyMonteCarloBarostatKernel::scaleCoordinates(ContextImpl& context, double scaleX, double scaleY, double scaleZ) {
8633
8634
8635
8636
8637
8638
8639
    if (!hasInitializedKernels) {
        hasInitializedKernels = true;

        // Create the arrays with the molecule definitions.

        vector<vector<int> > molecules = context.getMolecules();
        numMolecules = molecules.size();
peastman's avatar
peastman committed
8640
8641
8642
8643
        moleculeAtoms.initialize<int>(cl, cl.getNumAtoms(), "moleculeAtoms");
        moleculeStartIndex.initialize<int>(cl, numMolecules+1, "moleculeStartIndex");
        vector<int> atoms(moleculeAtoms.getSize());
        vector<int> startIndex(moleculeStartIndex.getSize());
8644
8645
8646
        int index = 0;
        for (int i = 0; i < numMolecules; i++) {
            startIndex[i] = index;
peastman's avatar
peastman committed
8647
8648
            for (int molecule : molecules[i])
                atoms[index++] = molecule;
8649
8650
        }
        startIndex[numMolecules] = index;
peastman's avatar
peastman committed
8651
8652
        moleculeAtoms.upload(atoms);
        moleculeStartIndex.upload(startIndex);
8653
8654
8655
8656

        // Initialize the kernel arguments.
        
        kernel.setArg<cl_int>(3, numMolecules);
8657
        kernel.setArg<cl::Buffer>(9, cl.getPosq().getDeviceBuffer());
peastman's avatar
peastman committed
8658
8659
        kernel.setArg<cl::Buffer>(10, moleculeAtoms.getDeviceBuffer());
        kernel.setArg<cl::Buffer>(11, moleculeStartIndex.getDeviceBuffer());
8660
    }
8661
    int bytesToCopy = cl.getPosq().getSize()*(cl.getUseDoublePrecision() ? sizeof(mm_double4) : sizeof(mm_float4));
peastman's avatar
peastman committed
8662
8663
    cl.getQueue().enqueueCopyBuffer(cl.getPosq().getDeviceBuffer(), savedPositions.getDeviceBuffer(), 0, 0, bytesToCopy);
    cl.getQueue().enqueueCopyBuffer(cl.getForce().getDeviceBuffer(), savedForces.getDeviceBuffer(), 0, 0, bytesToCopy);
8664
8665
8666
    kernel.setArg<cl_float>(0, (cl_float) scaleX);
    kernel.setArg<cl_float>(1, (cl_float) scaleY);
    kernel.setArg<cl_float>(2, (cl_float) scaleZ);
8667
    setPeriodicBoxArgs(cl, kernel, 4);
8668
    cl.executeKernel(kernel, cl.getNumAtoms());
peastman's avatar
peastman committed
8669
8670
    for (auto& offset : cl.getPosCellOffsets())
        offset = mm_int4(0, 0, 0, 0);
8671
    lastAtomOrder = cl.getAtomIndex();
8672
8673
8674
}

void OpenCLApplyMonteCarloBarostatKernel::restoreCoordinates(ContextImpl& context) {
8675
    int bytesToCopy = cl.getPosq().getSize()*(cl.getUseDoublePrecision() ? sizeof(mm_double4) : sizeof(mm_float4));
peastman's avatar
peastman committed
8676
8677
    cl.getQueue().enqueueCopyBuffer(savedPositions.getDeviceBuffer(), cl.getPosq().getDeviceBuffer(), 0, 0, bytesToCopy);
    cl.getQueue().enqueueCopyBuffer(savedForces.getDeviceBuffer(), cl.getForce().getDeviceBuffer(), 0, 0, bytesToCopy);
8678
8679
8680
8681
8682
}

void OpenCLRemoveCMMotionKernel::initialize(const System& system, const CMMotionRemover& force) {
    frequency = force.getFrequency();
    int numAtoms = cl.getNumAtoms();
peastman's avatar
peastman committed
8683
    cmMomentum.initialize<mm_float4>(cl, (numAtoms+OpenCLContext::ThreadBlockSize-1)/OpenCLContext::ThreadBlockSize, "cmMomentum");
8684
8685
8686
8687
    double totalMass = 0.0;
    for (int i = 0; i < numAtoms; i++)
        totalMass += system.getParticleMass(i);
    map<string, string> defines;
8688
    defines["INVERSE_TOTAL_MASS"] = cl.doubleToString(totalMass == 0 ? 0.0 : 1.0/totalMass);
8689
    cl::Program program = cl.createProgram(OpenCLKernelSources::removeCM, defines);
8690
8691
8692
    kernel1 = cl::Kernel(program, "calcCenterOfMassMomentum");
    kernel1.setArg<cl_int>(0, numAtoms);
    kernel1.setArg<cl::Buffer>(1, cl.getVelm().getDeviceBuffer());
peastman's avatar
peastman committed
8693
    kernel1.setArg<cl::Buffer>(2, cmMomentum.getDeviceBuffer());
8694
8695
8696
8697
    kernel1.setArg(3, OpenCLContext::ThreadBlockSize*sizeof(mm_float4), NULL);
    kernel2 = cl::Kernel(program, "removeCenterOfMassMomentum");
    kernel2.setArg<cl_int>(0, numAtoms);
    kernel2.setArg<cl::Buffer>(1, cl.getVelm().getDeviceBuffer());
peastman's avatar
peastman committed
8698
    kernel2.setArg<cl::Buffer>(2, cmMomentum.getDeviceBuffer());
8699
8700
8701
8702
8703
8704
8705
    kernel2.setArg(3, OpenCLContext::ThreadBlockSize*sizeof(mm_float4), NULL);
}

void OpenCLRemoveCMMotionKernel::execute(ContextImpl& context) {
    cl.executeKernel(kernel1, cl.getNumAtoms());
    cl.executeKernel(kernel2, cl.getNumAtoms());
}