Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
optimization.xml 19.4 KB
Newer Older
Davis King's avatar
Moved all the optimization stuff into its own page.  
Davis King committed
1
2
3
4
5
6
7
8
9
10
11
12
13 
<?xml version="1.0" encoding="ISO-8859-1"?>
<?xml-stylesheet type="text/xsl" href="stylesheet.xsl"?>

<doc>
   <title>Optimization</title>

   <!-- ************************************************************************* -->

   <body>
      <br/><br/>

         <p>
            This page documents library components that attempt to find the
Davis King's avatar
updated docs  
Davis King committed
14 
            minimum or maximum of a user supplied function.   An introduction
Davis King's avatar
updated the docs  
Davis King committed
15 
            to the general purpose non-linear optimizers in this section can be
Davis King's avatar
updated docs  
Davis King committed
16 
            found <a href="optimization_ex.cpp.html">here</a>.
Davis King's avatar
Moved all the optimization stuff into its own page.  
Davis King committed
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 
         </p>

   </body>

   <!-- ************************************************************************* -->

   <menu width="150">
    <top>
      <section>
         <name>Main Routines</name>
         <item>find_min</item> 
         <item>find_min_single_variable</item> 
         <item>find_min_using_approximate_derivatives</item> 
         <item>find_min_bobyqa</item> 
         <item>solve_qp_using_smo</item>
Davis King's avatar
updated the docs  
Davis King committed
32 
         <item>oca</item>
Davis King's avatar
Moved all the optimization stuff into its own page.  
Davis King committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307 
         <item>find_max</item> 
         <item>find_max_single_variable</item> 
         <item>find_max_using_approximate_derivatives</item> 
         <item>find_max_bobyqa</item> 
      </section>

      <section>
         <name>Strategies</name>
         <item>cg_search_strategy</item>
         <item>bfgs_search_strategy</item>
         <item>lbfgs_search_strategy</item>
         <item>objective_delta_stop_strategy</item>
         <item>gradient_norm_stop_strategy</item>
      </section>

      <section>
         <name>Helper Routines</name>
         <item>derivative</item> 
         <item>negate_function</item> 
         <item>make_line_search_function</item> 
         <item>poly_min_extrap</item> 
         <item>lagrange_poly_min_extrap</item> 
         <item>line_search</item> 
      </section>

    </top>  
   </menu>

   <!-- ************************************************************************* -->
   <!-- ************************************************************************* -->
   <!-- ************************************************************************* -->

   <components>
   
   <!-- ************************************************************************* -->
      
      <component>
         <name>derivative</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_abstract.h</spec_file>
         <description>
            This is a function that takes another function as input and returns
            a function object that numerically computes the derivative of the input function.
         </description>
                                 
      </component>


   <!-- ************************************************************************* -->
      
      <component>
         <name>negate_function</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_abstract.h</spec_file>
         <description>
            This is a function that takes another function as input and returns
            a function object that computes the negation of the input function.
         </description>
                                 
      </component>


   <!-- ************************************************************************* -->
      
      <component>
         <name>make_line_search_function</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_line_search_abstract.h</spec_file>
         <description>
            This is a function that takes another function f(x) as input and returns
            a function object l(z) = f(start + z*direction).   It is useful for
            turning multi-variable functions into single-variable functions for
            use with the <a href="#line_search">line_search</a> routine.
         </description>
                                 
      </component>


   <!-- ************************************************************************* -->
      
      <component>
         <name>poly_min_extrap</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_line_search_abstract.h</spec_file>
         <description>
            This function finds the 3rd degree polynomial that interpolates a 
            set of points and returns you the minimum of that polynomial.
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>lagrange_poly_min_extrap</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_line_search_abstract.h</spec_file>
         <description>
            This function finds the second order polynomial that interpolates a 
            set of points and returns you the minimum of that polynomial.
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>line_search</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_line_search_abstract.h</spec_file>
         <description>
            Performs a gradient based line search on a given function and returns the input
            that makes the function significantly smaller.
         </description>
                                 
      </component>


   <!-- ************************************************************************* -->

      <component>
         <name>cg_search_strategy</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_search_strategies_abstract.h</spec_file>
         <description>
                This object represents a strategy for determining which direction
                a <a href="#line_search">line search</a> should be carried out along.  This particular object
                is an implementation of the Polak-Ribiere conjugate gradient method
                for determining this direction.

                  <p>
                This method uses an amount of memory that is linear in the number
                of variables to be optimized.  So it is capable of handling problems
                with a very large number of variables.  However, it is generally
                not as good as the L-BFGS algorithm (see the 
                <a href="#lbfgs_search_strategy">lbfgs_search_strategy</a> class).
                  </p>
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>bfgs_search_strategy</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_search_strategies_abstract.h</spec_file>
         <description>
                This object represents a strategy for determining which direction
                a <a href="#line_search">line search</a> should be carried out along.  This particular object
                is an implementation of the BFGS quasi-newton method for determining 
                this direction.

                  <p>
                This method uses an amount of memory that is quadratic in the number
                of variables to be optimized.  It is generally very effective but 
                if your problem has a very large number of variables then it isn't 
                appropriate.  Instead You should try the <a href="#lbfgs_search_strategy">lbfgs_search_strategy</a>.
                  </p>
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>lbfgs_search_strategy</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_search_strategies_abstract.h</spec_file>
         <description>
                This object represents a strategy for determining which direction
                a <a href="#line_search">line search</a> should be carried out along.  This particular object
                is an implementation of the L-BFGS quasi-newton method for determining 
                this direction.

                  <p>
                This method uses an amount of memory that is linear in the number
                of variables to be optimized.  This makes it an excellent method 
                to use when an optimization problem has a large number of variables.
                  </p>
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>objective_delta_stop_strategy</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_stop_strategies_abstract.h</spec_file>
         <description>
                This object represents a strategy for deciding if an optimization
                algorithm should terminate.   This particular object looks at the 
                change in the objective function from one iteration to the next and 
                bases its decision on how large this change is.  If the change
                is below a user given threshold then the search stops.
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>gradient_norm_stop_strategy</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_stop_strategies_abstract.h</spec_file>
         <description>
                This object represents a strategy for deciding if an optimization
                algorithm should terminate.   This particular object looks at the 
                norm (i.e. the length) of the current gradient vector and
                stops if it is smaller than a user given threshold.  
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>find_min</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_abstract.h</spec_file>
         <description>
             Performs an unconstrained minimization of a nonlinear function using 
             some search strategy (e.g. <a href="#bfgs_search_strategy">bfgs_search_strategy</a>).
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->

      <component>
         <name>find_min_single_variable</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_line_search_abstract.h</spec_file>
         <description>
             Performs a bound constrained minimization of a nonlinear function.  The 
             function must be of a single variable.  Derivatives are not required.  
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->

      <component>
         <name>solve_qp_using_smo</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_solve_qp_using_smo_abstract.h</spec_file>
         <description>
             This function solves the following quadratic program:
<pre>
   Minimize: f(alpha) == 0.5*trans(alpha)*Q*alpha - trans(alpha)*b
   subject to the following constraints:
      sum(alpha) == C 
      min(alpha) >= 0 
</pre>

         </description>
                                 
      </component>
Davis King's avatar
updated the docs  
Davis King committed
308
309
310
311
312
313
314
315
316
317
318 
   <!-- ************************************************************************* -->
      
      <component>
         <name>oca</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_oca_abstract.h</spec_file>
         <description>
                This object is a tool for solving the following optimization problem:   
<pre>
   Minimize: f(w) == 0.5*dot(w,w) + C*R(w)
Davis King's avatar
updated docs  
Davis King committed
319 
   Where R(w) is a user-supplied convex function and C > 0
Davis King's avatar
updated the docs  
Davis King committed
320
321
322
323
324
325
326
327 
</pre>
<br/>
<br/>

                For a detailed discussion you should consult the following papers
                from the Journal of Machine Learning Research:
                <blockquote>
                    Optimized Cutting Plane Algorithm for Large-Scale Risk Minimization
Davis King's avatar
updated docs  
Davis King committed
328 
                      by  Vojtech Franc, Soren Sonnenburg; 10(Oct):2157--2192, 2009.
Davis King's avatar
updated the docs  
Davis King committed
329
330
331 
                </blockquote>
                <blockquote>
                    Bundle Methods for Regularized Risk Minimization
Davis King's avatar
updated docs  
Davis King committed
332 
                      by Choon Hui Teo, S.V.N. Vishwanthan, Alex J. Smola, Quoc V. Le; 11(Jan):311-365, 2010.
Davis King's avatar
updated the docs  
Davis King committed
333
334
335
336
337
338 
                </blockquote>

         </description>
                                 
      </component>
Davis King's avatar
Moved all the optimization stuff into its own page.  
Davis King committed
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473 
   <!-- ************************************************************************* -->
      
      
      <component>
         <name>find_min_bobyqa</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_bobyqa_abstract.h</spec_file>
         <description>
            This function defines the dlib interface to the BOBYQA software developed by M.J.D Powell.
            BOBYQA is a method for optimizing a function in the absence of derivative information.  
            Powell described it as a method that seeks the least value of a function of many 
            variables, by applying a trust region method that forms quadratic models by 
            interpolation.  There is usually some freedom in the interpolation conditions, 
            which is taken up by minimizing the Frobenius norm of the change to the second 
            derivative of the model, beginning with the zero matrix. The values of the variables 
            are constrained by upper and lower bounds.  

            <p>
            The following paper, published in 2009 by Powell, describes the
            detailed working of the BOBYQA algorithm.  

               <blockquote>
               The BOBYQA algorithm for bound constrained optimization 
               without derivatives by M.J.D. Powell
               </blockquote>
            </p>

            <p>
               Note that BOBYQA only works on functions of two or more variables.  So if you need to perform 
               derivative-free optimization on a function of a single variable 
               then you should use the <a href="#find_min_single_variable">find_min_single_variable</a>
               function.
            </p>

         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
            <example>model_selection_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>find_max_bobyqa</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_bobyqa_abstract.h</spec_file>
         <description>
            This function is identical to the <a href="#find_min_bobyqa">find_min_bobyqa</a> routine 
            except that it negates the objective function before performing optimization.  
            Thus this function will attempt to find the maximizer of the objective rather than 
            the minimizer.
            <p>
               Note that BOBYQA only works on functions of two or more variables.  So if you need to perform 
               derivative-free optimization on a function of a single variable 
               then you should use the <a href="#find_max_single_variable">find_max_single_variable</a>
               function.
            </p>
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
            <example>model_selection_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>find_min_using_approximate_derivatives</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_abstract.h</spec_file>
         <description>
             Performs an unconstrained minimization of a nonlinear function using 
             some search strategy (e.g. <a href="#bfgs_search_strategy">bfgs_search_strategy</a>).
             This version doesn't take a gradient function but instead numerically approximates 
             the gradient.
         </description>
         <examples>
            <example>optimization_ex.cpp.html</example>
         </examples>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>find_max</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_abstract.h</spec_file>
         <description>
             Performs an unconstrained maximization of a nonlinear function using 
             some search strategy (e.g. <a href="#bfgs_search_strategy">bfgs_search_strategy</a>).
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>find_max_single_variable</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_line_search_abstract.h</spec_file>
         <description>
             Performs a bound constrained maximization of a nonlinear function.  The 
             function must be of a single variable.  Derivatives are not required.
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->
      
      <component>
         <name>find_max_using_approximate_derivatives</name>
         <file>dlib/optimization.h</file>
         <spec_file link="true">dlib/optimization/optimization_abstract.h</spec_file>
         <description>
             Performs an unconstrained maximization of a nonlinear function using 
             some search strategy (e.g. <a href="#bfgs_search_strategy">bfgs_search_strategy</a>).
             This version doesn't take a gradient function but instead numerically approximates 
             the gradient.
         </description>
                                 
      </component>

   <!-- ************************************************************************* -->

   </components>

   <!-- ************************************************************************* -->


</doc>