Update files

cpp/thirdpart/kenlm/cmake/KenLMFunctions.cmake

+# Helper functions used across the CMake build system
+
+include(CMakeParseArguments)
+
+# Adds a bunch of executables to the build, each depending on the specified
+# dependent object files and linking against the specified libraries
+function(AddExes)
+  set(multiValueArgs EXES DEPENDS LIBRARIES)
+  cmake_parse_arguments(AddExes "" "" "${multiValueArgs}" ${ARGN})
+
+  # Iterate through the executable list
+  foreach(exe ${AddExes_EXES})
+
+    # Compile the executable, linking against the requisite dependent object files
+    add_executable(${exe} ${exe}_main.cc ${AddExes_DEPENDS})
+
+    # Link the executable against the supplied libraries
+    target_link_libraries(${exe} ${AddExes_LIBRARIES})
+
+    # Group executables together
+    set_target_properties(${exe} PROPERTIES FOLDER executables)
+
+  # End for loop
+  endforeach(exe)
+
+  # Install the executable files
+  install(TARGETS ${AddExes_EXES} DESTINATION bin)
+endfunction()
+
+# Adds a single test to the build, depending on the specified dependent
+# object files, linking against the specified libraries, and with the
+# specified command line arguments
+function(KenLMAddTest)
+  cmake_parse_arguments(KenLMAddTest "" "TEST"
+                        "DEPENDS;LIBRARIES;TEST_ARGS" ${ARGN})
+
+  # Compile the executable, linking against the requisite dependent object files
+  add_executable(${KenLMAddTest_TEST}
+                 ${KenLMAddTest_TEST}.cc
+                 ${KenLMAddTest_DEPENDS})
+
+  if (Boost_USE_STATIC_LIBS)
+    set(DYNLINK_FLAGS)
+  else()
+    set(DYNLINK_FLAGS COMPILE_FLAGS -DBOOST_TEST_DYN_LINK)
+  endif()
+
+  # Require the following compile flag
+  set_target_properties(${KenLMAddTest_TEST} PROPERTIES
+                        ${DYNLINK_FLAGS}
+                        RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/tests)
+
+  target_link_libraries(${KenLMAddTest_TEST} ${KenLMAddTest_LIBRARIES} ${TIMER_LINK})
+
+  set(test_params "")
+  if(KenLMAddTest_TEST_ARGS)
+    set(test_params ${KenLMAddTest_TEST_ARGS})
+  endif()
+
+  # Specify command arguments for how to run each unit test
+  add_test(NAME ${KenLMAddTest_TEST}
+           COMMAND ${KenLMAddTest_TEST} ${test_params})
+
+  # Group unit tests together
+  set_target_properties(${KenLMAddTest_TEST} PROPERTIES FOLDER "unit_tests")
+endfunction()
+
+# Adds a bunch of tests to the build, each depending on the specified
+# dependent object files and linking against the specified libraries
+function(AddTests)
+  set(multiValueArgs TESTS DEPENDS LIBRARIES TEST_ARGS)
+  cmake_parse_arguments(AddTests "" "" "${multiValueArgs}" ${ARGN})
+
+  # Iterate through the Boost tests list
+  foreach(test ${AddTests_TESTS})
+    KenLMAddTest(TEST ${test}
+                 DEPENDS ${AddTests_DEPENDS}
+                 LIBRARIES ${AddTests_LIBRARIES}
+                 TEST_ARGS ${AddTests_TEST_ARGS})
+  endforeach(test)
+endfunction()

cpp/thirdpart/kenlm/cmake/kenlmConfig.cmake.in

+@PACKAGE_INIT@
+
+include(CMakeFindDependencyMacro)
+
+find_dependency(Boost)
+find_dependency(Threads)
+
+# Compression libs
+if (@ZLIB_FOUND@)
+  find_dependency(ZLIB)
+endif()
+if (@BZIP2_FOUND@)
+  find_dependency(BZip2)
+endif()
+if (@LIBLZMA_FOUND@)
+  find_dependency(LibLZMA)
+endif()
+
+include("${CMAKE_CURRENT_LIST_DIR}/kenlmTargets.cmake")

cpp/thirdpart/kenlm/cmake/modules/FindEigen3.cmake

+# - Try to find Eigen3 lib
+#
+# This module supports requiring a minimum version, e.g. you can do
+#   find_package(Eigen3 3.1.2)
+# to require version 3.1.2 or newer of Eigen3.
+#
+# Once done this will define
+#
+#  EIGEN3_FOUND - system has eigen lib with correct version
+#  EIGEN3_INCLUDE_DIR - the eigen include directory
+#  EIGEN3_VERSION - eigen version
+#
+# This module reads hints about search locations from 
+# the following enviroment variables:
+#
+# EIGEN3_ROOT
+# EIGEN3_ROOT_DIR
+
+# Copyright (c) 2006, 2007 Montel Laurent, <montel@kde.org>
+# Copyright (c) 2008, 2009 Gael Guennebaud, <g.gael@free.fr>
+# Copyright (c) 2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+# Redistribution and use is allowed according to the terms of the 2-clause BSD license.
+
+if(NOT Eigen3_FIND_VERSION)
+  if(NOT Eigen3_FIND_VERSION_MAJOR)
+    set(Eigen3_FIND_VERSION_MAJOR 2)
+  endif(NOT Eigen3_FIND_VERSION_MAJOR)
+  if(NOT Eigen3_FIND_VERSION_MINOR)
+    set(Eigen3_FIND_VERSION_MINOR 91)
+  endif(NOT Eigen3_FIND_VERSION_MINOR)
+  if(NOT Eigen3_FIND_VERSION_PATCH)
+    set(Eigen3_FIND_VERSION_PATCH 0)
+  endif(NOT Eigen3_FIND_VERSION_PATCH)
+
+  set(Eigen3_FIND_VERSION "${Eigen3_FIND_VERSION_MAJOR}.${Eigen3_FIND_VERSION_MINOR}.${Eigen3_FIND_VERSION_PATCH}")
+endif(NOT Eigen3_FIND_VERSION)
+
+macro(_eigen3_check_version)
+  file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header)
+
+  string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}")
+  set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}")
+  string(REGEX MATCH "define[ \t]+EIGEN_MAJOR_VERSION[ \t]+([0-9]+)" _eigen3_major_version_match "${_eigen3_version_header}")
+  set(EIGEN3_MAJOR_VERSION "${CMAKE_MATCH_1}")
+  string(REGEX MATCH "define[ \t]+EIGEN_MINOR_VERSION[ \t]+([0-9]+)" _eigen3_minor_version_match "${_eigen3_version_header}")
+  set(EIGEN3_MINOR_VERSION "${CMAKE_MATCH_1}")
+
+  set(EIGEN3_VERSION ${EIGEN3_WORLD_VERSION}.${EIGEN3_MAJOR_VERSION}.${EIGEN3_MINOR_VERSION})
+  if(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION})
+    set(EIGEN3_VERSION_OK FALSE)
+  else(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION})
+    set(EIGEN3_VERSION_OK TRUE)
+  endif(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION})
+
+  if(NOT EIGEN3_VERSION_OK)
+
+    message(STATUS "Eigen3 version ${EIGEN3_VERSION} found in ${EIGEN3_INCLUDE_DIR}, "
+                   "but at least version ${Eigen3_FIND_VERSION} is required")
+  endif(NOT EIGEN3_VERSION_OK)
+endmacro(_eigen3_check_version)
+
+if (EIGEN3_INCLUDE_DIR)
+
+  # in cache already
+  _eigen3_check_version()
+  set(EIGEN3_FOUND ${EIGEN3_VERSION_OK})
+
+else (EIGEN3_INCLUDE_DIR)
+
+  find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library
+      HINTS
+      ENV EIGEN3_ROOT 
+      ENV EIGEN3_ROOT_DIR
+      PATHS
+      ${CMAKE_INSTALL_PREFIX}/include
+      ${KDE4_INCLUDE_DIR}
+      PATH_SUFFIXES eigen3 eigen
+    )
+
+  if(EIGEN3_INCLUDE_DIR)
+    _eigen3_check_version()
+  endif(EIGEN3_INCLUDE_DIR)
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Eigen3 DEFAULT_MSG EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK)
+
+  mark_as_advanced(EIGEN3_INCLUDE_DIR)
+
+endif(EIGEN3_INCLUDE_DIR)
+

cpp/thirdpart/kenlm/compile_query_only.sh

+#!/bin/bash
+#This is just an example compilation.  You should integrate these files into your build system.  Boost jam is provided and preferred.
+
+echo You must use ./bjam if you want language model estimation, filtering, or support for compressed files \(.gz, .bz2, .xz\) 1>&2
+
+rm {lm,util}/*.o 2>/dev/null
+set -e
+
+CXX=${CXX:-g++}
+
+CXXFLAGS+=" -I. -O3 -DNDEBUG -DKENLM_MAX_ORDER=6"
+
+#If this fails for you, consider using bjam.
+if [ ${#NPLM} != 0 ]; then
+  CXXFLAGS+=" -DHAVE_NPLM -lneuralLM -L$NPLM/src -I$NPLM/src -lboost_thread-mt -fopenmp"
+  ADDED_PATHS="lm/wrappers/*.cc"
+fi
+echo 'Compiling with '$CXX $CXXFLAGS
+
+#Grab all cc files in these directories except those ending in test.cc or main.cc
+objects=""
+for i in util/double-conversion/*.cc util/*.cc lm/*.cc $ADDED_PATHS; do
+  if [ "${i%test.cc}" == "$i" ] && [ "${i%main.cc}" == "$i" ]; then
+    $CXX $CXXFLAGS -c $i -o ${i%.cc}.o
+    objects="$objects ${i%.cc}.o"
+  fi
+done
+
+mkdir -p bin
+if [ "$(uname)" != Darwin ]; then
+  CXXFLAGS="$CXXFLAGS -lrt"
+fi
+$CXX lm/build_binary_main.cc $objects -o bin/build_binary $CXXFLAGS $LDFLAGS
+$CXX lm/query_main.cc $objects -o bin/query $CXXFLAGS $LDFLAGS

cpp/thirdpart/kenlm/lm/builder/TODO

+More tests!
+Sharding.
+Some way to manage all the crazy config options.
+Option to build the binary file directly.  
+Interpolation of different orders.