Initial commit

Src/Utility/Filesystem.cpp

+#include <Filesystem.h>
+#include <algorithm>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fstream>
+#ifdef _WIN32
+#include <io.h>
+#include <direct.h>
+#include <Windows.h>
+#else
+#include <unistd.h>
+#include <dirent.h>
+#endif
+
+// 路径分隔符(Linux:‘/’,Windows:’\\’)
+#ifdef _WIN32
+#define  PATH_SEPARATOR '\\'
+#else
+#define  PATH_SEPARATOR '/'
+#endif
+
+using namespace std;
+
+namespace ortSamples
+{
+
+static std::vector<std::string> SplitString(std::string str, std::string separator)
+{
+    std::string::size_type pos;
+    std::vector<std::string> result;
+    str+=separator;//扩展字符串以方便操作
+    int size=str.size();
+
+    for(int i=0; i<size; i++)
+    {
+        pos=str.find(separator,i);
+        if(pos<size)
+        {
+            std::string s=str.substr(i,pos-i);
+            result.push_back(s);
+            i=pos+separator.size()-1;
+        }
+    }
+    return result;
+}
+
+#if defined _WIN32 || defined WINCE
+    const char dir_separators[] = "/\\";
+
+	struct dirent
+	{
+		const char* d_name;
+	};
+
+	struct DIR
+	{
+#ifdef WINRT
+		WIN32_FIND_DATAW data;
+#else
+        WIN32_FIND_DATAA data;
+#endif
+		HANDLE handle;
+		dirent ent;
+#ifdef WINRT
+		DIR() { }
+		~DIR()
+		{
+			if (ent.d_name)
+				delete[] ent.d_name;
+		}
+#endif
+	};
+
+	DIR* opendir(const char* path)
+	{
+		DIR* dir = new DIR;
+		dir->ent.d_name = 0;
+#ifdef WINRT
+		string full_path = string(path) + "\\*";
+		wchar_t wfull_path[MAX_PATH];
+		size_t copied = mbstowcs(wfull_path, full_path.c_str(), MAX_PATH);
+		CV_Assert((copied != MAX_PATH) && (copied != (size_t)-1));
+		dir->handle = ::FindFirstFileExW(wfull_path, FindExInfoStandard,
+			&dir->data, FindExSearchNameMatch, NULL, 0);
+#else
+		dir->handle = ::FindFirstFileExA((string(path) + "\\*").c_str(),
+			FindExInfoStandard, &dir->data, FindExSearchNameMatch, NULL, 0);
+#endif
+		if (dir->handle == INVALID_HANDLE_VALUE)
+		{
+			/*closedir will do all cleanup*/
+			delete dir;
+			return 0;
+		}
+		return dir;
+	}
+
+	dirent* readdir(DIR* dir)
+	{
+#ifdef WINRT
+		if (dir->ent.d_name != 0)
+		{
+			if (::FindNextFileW(dir->handle, &dir->data) != TRUE)
+				return 0;
+		}
+		size_t asize = wcstombs(NULL, dir->data.cFileName, 0);
+		CV_Assert((asize != 0) && (asize != (size_t)-1));
+		char* aname = new char[asize + 1];
+		aname[asize] = 0;
+		wcstombs(aname, dir->data.cFileName, asize);
+		dir->ent.d_name = aname;
+#else
+		if (dir->ent.d_name != 0)
+		{
+			if (::FindNextFileA(dir->handle, &dir->data) != TRUE)
+				return 0;
+		}
+		dir->ent.d_name = dir->data.cFileName;
+#endif
+		return &dir->ent;
+	}
+
+	void closedir(DIR* dir)
+	{
+		::FindClose(dir->handle);
+		delete dir;
+	}
+#else
+# include <dirent.h>
+# include <sys/stat.h>
+	const char dir_separators[] = "/";
+#endif
+
+	static bool isDir(const string &path, DIR* dir)
+	{
+#if defined _WIN32 || defined WINCE
+		DWORD attributes;
+		BOOL status = TRUE;
+		if (dir)
+			attributes = dir->data.dwFileAttributes;
+		else
+		{
+			WIN32_FILE_ATTRIBUTE_DATA all_attrs;
+#ifdef WINRT
+			wchar_t wpath[MAX_PATH];
+			size_t copied = mbstowcs(wpath, path.c_str(), MAX_PATH);
+			CV_Assert((copied != MAX_PATH) && (copied != (size_t)-1));
+			status = ::GetFileAttributesExW(wpath, GetFileExInfoStandard, &all_attrs);
+#else
+			status = ::GetFileAttributesExA(path.c_str(), GetFileExInfoStandard, &all_attrs);
+#endif
+			attributes = all_attrs.dwFileAttributes;
+		}
+
+		return status && ((attributes & FILE_ATTRIBUTE_DIRECTORY) != 0);
+#else
+		(void)dir;
+		struct stat stat_buf;
+		if (0 != stat(path.c_str(), &stat_buf))
+			return false;
+		int is_dir = S_ISDIR(stat_buf.st_mode);
+		return is_dir != 0;
+#endif
+	}
+
+	bool IsDirectory(const string &path)
+	{
+		return isDir(path, NULL);
+	}
+
+	bool Exists(const string& path)
+	{
+
+#if defined _WIN32 || defined WINCE
+			BOOL status = TRUE;
+		{
+			WIN32_FILE_ATTRIBUTE_DATA all_attrs;
+#ifdef WINRT
+			wchar_t wpath[MAX_PATH];
+			size_t copied = mbstowcs(wpath, path.c_str(), MAX_PATH);
+			CV_Assert((copied != MAX_PATH) && (copied != (size_t)-1));
+			status = ::GetFileAttributesExW(wpath, GetFileExInfoStandard, &all_attrs);
+#else
+			status = ::GetFileAttributesExA(path.c_str(), GetFileExInfoStandard, &all_attrs);
+#endif
+		}
+
+		return !!status;
+#else
+		struct stat stat_buf;
+		return (0 == stat(path.c_str(), &stat_buf));
+#endif
+	}
+
+    bool IsPathSeparator(char c)
+    {
+        return c == '/' || c == '\\';
+	}
+
+	string JoinPath(const string& base, const string& path)
+	{
+		if (base.empty())
+			return path;
+		if (path.empty())
+			return base;
+
+		bool baseSep = IsPathSeparator(base[base.size() - 1]);
+		bool pathSep = IsPathSeparator(path[0]);
+		string result;
+		if (baseSep && pathSep)
+		{
+			result = base + path.substr(1);
+		}
+		else if (!baseSep && !pathSep)
+		{
+            result = base + PATH_SEPARATOR + path;
+		}
+		else
+		{
+			result = base + path;
+		}
+		return result;
+	}
+
+	static bool wildcmp(const char *string, const char *wild)
+	{
+		const char *cp = 0, *mp = 0;
+
+		while ((*string) && (*wild != '*'))
+		{
+			if ((*wild != *string) && (*wild != '?'))
+			{
+				return false;
+			}
+
+			wild++;
+			string++;
+		}
+
+		while (*string)
+		{
+			if (*wild == '*')
+			{
+				if (!*++wild)
+				{
+					return true;
+				}
+
+				mp = wild;
+				cp = string + 1;
+			}
+			else if ((*wild == *string) || (*wild == '?'))
+			{
+				wild++;
+				string++;
+			}
+			else
+			{
+				wild = mp;
+				string = cp++;
+			}
+		}
+
+		while (*wild == '*')
+		{
+			wild++;
+		}
+
+		return *wild == 0;
+	}
+
+	static void glob_rec(const string &directory, const string& wildchart, std::vector<string>& result,
+		bool recursive, bool includeDirectories, const string& pathPrefix)
+	{
+		DIR *dir;
+
+		if ((dir = opendir(directory.c_str())) != 0)
+		{
+			/* find all the files and directories within directory */
+			try
+			{
+				struct dirent *ent;
+				while ((ent = readdir(dir)) != 0)
+				{
+					const char* name = ent->d_name;
+					if ((name[0] == 0) || (name[0] == '.' && name[1] == 0) || (name[0] == '.' && name[1] == '.' && name[2] == 0))
+						continue;
+
+					string path = JoinPath(directory, name);
+					string entry = JoinPath(pathPrefix, name);
+
+					if (isDir(path, dir))
+					{
+						if (recursive)
+							glob_rec(path, wildchart, result, recursive, includeDirectories, entry);
+						if (!includeDirectories)
+							continue;
+					}
+
+					if (wildchart.empty() || wildcmp(name, wildchart.c_str()))
+						result.push_back(entry);
+				}
+			}
+			catch (...)
+			{
+				closedir(dir);
+				throw;
+			}
+			closedir(dir);
+		}
+		else
+		{
+			printf("could not open directory: %s", directory.c_str());
+		}
+	}
+
+	void GetFileNameList(const string &directory, const string &pattern, std::vector<string>& result, bool recursive, bool addPath)
+	{
+        // split pattern
+        vector<string> patterns=SplitString(pattern,",");
+
+        result.clear();
+
+        for(int i=0;i<patterns.size();++i)
+        {
+            string eachPattern=patterns[i];
+            std::vector<string> eachResult;
+            glob_rec(directory, eachPattern, eachResult, recursive, true, directory);
+            for(int j=0;j<eachResult.size();++j)
+            {
+				if (IsDirectory(eachResult[j]))
+					continue;
+                if(addPath)
+                {
+                    result.push_back(eachResult[j]);
+                }
+                else
+                {
+                    result.push_back(GetFileName(eachResult[j]));
+                }
+            }
+        }
+		std::sort(result.begin(), result.end());
+	}
+
+	void GetFileNameList2(const string &directory, const string &pattern, std::vector<string>& result, bool recursive, bool addPath)
+	{
+		// split pattern
+		vector<string> patterns = SplitString(pattern, ",");
+
+        result.clear();
+
+		for (int i = 0; i<patterns.size(); ++i)
+		{
+			string eachPattern = patterns[i];
+			std::vector<string> eachResult;
+			glob_rec(directory, eachPattern, eachResult, recursive, true, directory);
+			for (int j = 0; j<eachResult.size(); ++j)
+			{
+				string filePath = eachResult[j];
+				if (IsDirectory(filePath))
+				{
+					filePath = filePath + "/";
+					for (int k = 0; k < filePath.size(); ++k)
+					{
+						if (IsPathSeparator(filePath[k]))
+						{
+							filePath[k] = '/';
+						}
+					}
+				}
+				if (addPath)
+				{
+					result.push_back(filePath);
+				}
+				else
+				{
+					if (!IsDirectory(filePath))
+					{
+						result.push_back(GetFileName(filePath));
+					}
+				}
+			}
+		}
+		std::sort(result.begin(), result.end());
+	}
+
+	void RemoveAll(const string& path)
+	{
+
+		if (!Exists(path))
+			return;
+
+		if (IsDirectory(path))
+		{
+			std::vector<string> entries;
+			GetFileNameList2(path, string(), entries, false, true);
+			for (size_t i = 0; i < entries.size(); i++)
+			{
+				const string& e = entries[i];
+				RemoveAll(e);
+			}
+#ifdef _MSC_VER
+			bool result = _rmdir(path.c_str()) == 0;
+#else
+			bool result = rmdir(path.c_str()) == 0;
+#endif
+			if (!result)
+			{
+				printf("can't remove directory: %s\n", path.c_str());
+			}
+		}
+		else
+		{
+#ifdef _MSC_VER
+			bool result = _unlink(path.c_str()) == 0;
+#else
+			bool result = unlink(path.c_str()) == 0;
+#endif
+			if (!result)
+			{
+				printf("can't remove file: %s\n", path.c_str());
+			}
+		}
+	}
+
+	void Remove(const string &directory, const string &extension)
+	{
+
+		DIR *dir;
+
+		static int numberOfFiles = 0;
+
+		if ((dir = opendir(directory.c_str())) != 0)
+		{
+			/* find all the files and directories within directory */
+			try
+			{
+				struct dirent *ent;
+				while ((ent = readdir(dir)) != 0)
+				{
+					const char* name = ent->d_name;
+					if ((name[0] == 0) || (name[0] == '.' && name[1] == 0) || (name[0] == '.' && name[1] == '.' && name[2] == 0))
+						continue;
+
+					string path = JoinPath(directory, name);
+
+					if (isDir(path, dir))
+					{
+						Remove(path, extension);
+					}
+
+					// �ж���չ��
+					if (extension.empty() || wildcmp(name, extension.c_str()))
+					{
+						RemoveAll(path);
+						++numberOfFiles;
+						printf("%s deleted! number of deleted files:%d\n", path.c_str(), numberOfFiles);
+					}
+
+				}
+			}
+			catch (...)
+			{
+				closedir(dir);
+				throw;
+			}
+			closedir(dir);
+		}
+		else
+		{
+			printf("could not open directory: %s", directory.c_str());
+		}
+
+		// ����RemoveAllɾ��Ŀ¼
+		RemoveAll(directory);
+	}
+	string GetFileName(const string &path)
+	{
+        string fileName;
+        int indexOfPathSeparator = -1;
+        for (int i = path.size() - 1; i >= 0; --i)
+        {
+            if (IsPathSeparator(path[i]))
+            {
+                fileName = path.substr(i + 1, path.size() - i - 1);
+                indexOfPathSeparator = i;
+                break;
+            }
+        }
+        if (indexOfPathSeparator == -1)
+        {
+            fileName = path;
+        }
+
+        return fileName;
+	}
+    string GetFileName_NoExtension(const string &path)
+    {
+        string fileName=GetFileName(path);
+        string fileName_NoExtension;
+        for(int i=fileName.size()-1;i>0;--i)
+        {
+            if(fileName[i]=='.')
+            {
+                fileName_NoExtension=fileName.substr(0,i);
+                break;
+            }
+        }
+
+        return fileName_NoExtension;
+    }
+
+	string GetExtension(const string &path)
+	{
+		string fileName;
+		for (int i = path.size() - 1; i >= 0; --i)
+		{
+			if (path[i]=='.')
+			{
+				fileName = path.substr(i, path.size() - i);
+				break;
+			}
+		}
+
+		return fileName;
+
+	}
+
+	string GetParentPath(const string &path)
+	{
+		string fileName;
+		for (int i = path.size() - 1; i >= 0; --i)
+		{
+			if (IsPathSeparator(path[i]))
+			{
+				fileName = path.substr(0, i+1);
+				break;
+			}
+		}
+
+		return fileName;
+	}
+
+    static bool CreateDirectory(const string &path)
+    {
+    #if defined WIN32 || defined _WIN32 || defined WINCE
+    #ifdef WINRT
+                wchar_t wpath[MAX_PATH];
+            size_t copied = mbstowcs(wpath, path.c_str(), MAX_PATH);
+            CV_Assert((copied != MAX_PATH) && (copied != (size_t)-1));
+            int result = CreateDirectoryA(wpath, NULL) ? 0 : -1;
+    #else
+                int result = _mkdir(path.c_str());
+    #endif
+    #elif defined __linux__ || defined __APPLE__
+                int result = mkdir(path.c_str(), 0777);
+    #else
+                int result = -1;
+    #endif
+
+            if (result == -1)
+            {
+                return IsDirectory(path);
+            }
+            return true;
+    }
+
+	bool CreateDirectories(const string &directoryPath)
+	{
+		string path = directoryPath;
+
+		for (;;)
+		{
+			char last_char = path.empty() ? 0 : path[path.length() - 1];
+			if (IsPathSeparator(last_char))
+			{
+				path = path.substr(0, path.length() - 1);
+				continue;
+			}
+			break;
+		}
+
+		if (path.empty() || path == "./" || path == ".\\" || path == ".")
+			return true;
+		if (IsDirectory(path))
+			return true;
+
+		size_t pos = path.rfind('/');
+		if (pos == string::npos)
+			pos = path.rfind('\\');
+		if (pos != string::npos)
+		{
+			string parent_directory = path.substr(0, pos);
+			if (!parent_directory.empty())
+			{
+				if (!CreateDirectories(parent_directory))
+					return false;
+			}
+		}
+
+		return CreateDirectory(path);
+	}
+
+    bool CopyFile(const string srcPath, const string dstPath)
+    {
+        std::ifstream srcFile(srcPath,ios::binary);
+        std::ofstream dstFile(dstPath,ios::binary);
+
+        if(!srcFile.is_open())
+        {
+            printf("can not open %s\n",srcPath.c_str());
+            return false;
+        }
+        if(!dstFile.is_open())
+        {
+			printf("can not open %s\n", dstPath.c_str());
+            return false;
+        }
+        if(srcPath==dstPath)
+        {
+			printf("src can not be same with dst\n");
+            return false;
+        }
+        char buffer[2048];
+        unsigned int numberOfBytes=0;
+        while(srcFile)
+        {
+            srcFile.read(buffer,2048);
+            dstFile.write(buffer,srcFile.gcount());
+            numberOfBytes+=srcFile.gcount();
+        }
+        srcFile.close();
+        dstFile.close();
+        return true;
+    }
+
+    bool CopyDirectories(string srcPath, const string dstPath)
+    {
+        if(srcPath==dstPath)
+        {
+			printf("src can not be same with dst\n");
+            return false;
+        }
+
+		// ȥ������·���ָ���
+		srcPath = srcPath.substr(0, srcPath.size() - 1);
+
+        vector<string> fileNameList;
+        GetFileNameList2(srcPath, "", fileNameList, true, true);
+
+        string parentPathOfSrc=GetParentPath(srcPath);
+        int length=parentPathOfSrc.size();
+
+        // create all directories
+        for(int i=0;i<fileNameList.size();++i)
+        {
+            // create directory
+            string srcFilePath=fileNameList[i];
+            string subStr=srcFilePath.substr(length,srcFilePath.size()-length);
+            string dstFilePath=dstPath+subStr;
+            string parentPathOfDst=GetParentPath(dstFilePath);
+            CreateDirectories(parentPathOfDst);
+        }
+
+        // copy file
+        for(int i=0;i<fileNameList.size();++i)
+        {
+            string srcFilePath=fileNameList[i];
+			if (IsDirectory(srcFilePath))
+			{
+				continue;
+			}
+            string subStr=srcFilePath.substr(length,srcFilePath.size()-length);
+            string dstFilePath=dstPath+subStr;
+
+            // copy file
+            CopyFile(srcFilePath,dstFilePath);
+
+			// process
+			double process = (1.0*(i + 1) / fileNameList.size()) * 100;
+			printf("%s done! %f% \n", GetFileName(fileNameList[i]).c_str(), process);
+        }
+		printf("all done!(the number of files:%d)\n", fileNameList.size());
+
+        return true;
+
+
+    }
+
+}
+
+

Src/Utility/Filesystem.h

+// 文件以及目录处理
+
+#ifndef __FILE_SYSTEM_H__
+#define __FILE_SYSTEM_H__
+
+#include <string>
+#include <vector>
+ 
+namespace ortSamples
+{
+
+// 路径是否存在
+bool Exists(const std::string &path);
+
+// 路径是否为目录
+bool IsDirectory(const std::string &path);
+
+// 是否是路径分隔符(Linux:‘/’,Windows:’\\’)
+bool IsPathSeparator(char c);
+
+// 路径拼接
+std::string JoinPath(const std::string &base, const std::string &path);
+
+// 创建多级目录,注意：创建多级目录的时候，目标目录是不能有文件存在的
+bool CreateDirectories(const std::string &directoryPath);
+
+/** 生成符合指定模式的文件名列表(支持递归遍历)
+* 
+* pattern: 模式,比如"*.jpg","*.png","*.jpg,*.png"
+* addPath：是否包含父路径
+* 注意：
+    1. 多个模式使用","分割,比如"*.jpg,*.png"
+    2. 支持通配符'*','?' ,比如第一个字符是7的所有文件名:"7*.*", 以512结尾的所有jpg文件名："*512.jpg"
+    3. 使用"*.jpg"，而不是".jpg"
+    4. 空string表示返回所有结果
+    5. 不能返回子目录名
+*
+*/
+void GetFileNameList(const std::string &directory, const std::string &pattern, std::vector<std::string> &result, bool recursive, bool addPath);
+
+// 与GetFileNameList的区别在于如果有子目录，在addPath为true的时候会返回子目录路径(目录名最后有"/")
+void GetFileNameList2(const std::string &directory, const std::string &pattern, std::vector<std::string> &result, bool recursive, bool addPath);
+
+// 删除文件或者目录,支持递归删除
+void Remove(const std::string &directory, const std::string &extension="");
+
+/** 获取路径的文件名和扩展名
+ * 
+ *  示例：path为D:/1/1.txt,则GetFileName()为1.txt,GetFileName_NoExtension()为1,GetExtension()为.txt,GetParentPath()为D:/1/
+*/
+std::string GetFileName(const std::string &path);
+std::string GetFileName_NoExtension(const std::string &path); 
+std::string GetExtension(const std::string &path);
+std::string GetParentPath(const std::string &path);
+
+// 拷贝文件
+bool CopyFile(const std::string srcPath,const std::string dstPath);
+
+/** 拷贝目录
+ * 
+ * 示例：CopyDirectories("D:/0/1/2/","E:/3/");实现把D:/0/1/2/目录拷贝到E:/3/目录中(即拷贝完成后的目录结构为E:/3/2/)
+ * 注意：
+    1.第一个参数的最后不能加”/”
+    2.不能拷贝隐藏文件
+*/
+bool CopyDirectories(std::string srcPath,const std::string dstPath);
+
+}
+
+#endif

Src/Utility/SimpleLog.h

+// 简易日志
+
+#ifndef __SIMPLE_LOG_H__
+#define __SIMPLE_LOG_H__
+
+#include <time.h>
+#include <string>
+#include <map>
+#include <thread>
+#include <mutex>
+#if (defined WIN32 || defined _WIN32)
+#include <Windows.h>
+#else
+#include <sys/time.h>
+#endif
+
+using namespace std;
+
+
+/** 简易日志
+ * 
+ * 不依赖于其他第三方库，只需要包含一个头文件就可以使用。提供了4种日志级别，包括INFO,DEBUG,WARN和ERROR。
+ * 
+ * 示例1:
+    // 初始化日志，在./Log/目录下创建两个日志文件log1.log和log2.log(注意：目录./Log/需要存在，否则日志创建失败)
+    LogManager::GetInstance()->Initialize("./Log/","log1");
+    LogManager::GetInstance()->Initialize("./Log/","log2");
+
+    // 写日志
+    string log = "Hello World";
+    LOG_INFO(LogManager::GetInstance()->GetLogFile("log1"), "%s\n", log.c_str()); // 写入log1.log
+    LOG_INFO(LogManager::GetInstance()->GetLogFile("log2"), "%s\n", log.c_str()); // 写入log2.log
+
+    // 关闭日志
+    LogManager::GetInstance()->Close("log1");
+    LogManager::GetInstance()->Close("log2");
+  
+  * 示例2:
+    // 将日志输出到控制台
+    string log = "Hello World";
+    LOG_INFO(stdout, "%s\n", log.c_str()); 
+
+  * 注意：
+    1. 需要C++11
+    2. 多线程的时候需要加锁(打开#define LOG_MUTEX)，否则会导致日志显示混乱
+
+*/
+
+// #define LOG_MUTEX  // 加锁
+
+class LogManager
+{
+private:
+    LogManager(){}
+
+public:
+    ~LogManager(){}
+    inline void Initialize(const string &parentPath,const string &logName)
+    {
+        // 日志名为空表示输出到控制台
+        if(logName.size()==0)
+            return;
+
+        // 查找该日志文件，如果没有则创建
+        std::map<string, FILE*>::const_iterator iter = logMap.find(logName);
+        if (iter == logMap.end())
+        {
+            string pathOfLog = parentPath+ logName + ".log";
+            FILE *logFile = fopen(pathOfLog.c_str(), "a"); // w:覆盖原有文件,a:追加
+            if(logFile!=NULL)
+            {
+                logMap.insert(std::make_pair(logName, logFile));
+            }
+        }
+
+    }
+    inline FILE* GetLogFile(const string &logName)
+    {
+        std::map<string, FILE*>::const_iterator iter=logMap.find(logName);
+        if(iter==logMap.end())
+        {
+            return NULL;
+        }
+
+        return (*iter).second;
+    }
+    inline void Close(const string &logName)
+    {
+        std::map<string, FILE*>::const_iterator iter=logMap.find(logName);
+        if(iter==logMap.end())
+        {
+            return;
+        }
+
+        fclose((*iter).second);
+        logMap.erase(iter);
+    }
+    inline std::mutex &GetLogMutex()
+    {
+        return logMutex;
+    }
+
+    // Singleton
+    static LogManager* GetInstance()
+    {
+        static LogManager logManager;
+        return &logManager;
+    }
+private:
+    std::map<string, FILE*> logMap;
+    std::mutex logMutex;
+};
+
+#ifdef LOG_MUTEX
+    #define LOCK	LogManager::GetInstance()->GetLogMutex().lock()
+    #define UNLOCK	LogManager::GetInstance()->GetLogMutex().unlock()
+#else
+    #define LOCK
+    #define UNLOCK
+#endif
+
+// log time
+typedef struct  _LogTime
+{
+    string year;
+    string month;
+    string day;
+    string hour;
+    string minute;
+    string second;
+    string millisecond; // ms
+    string microsecond; // us
+    string weekDay;
+}LogTime;
+
+inline LogTime GetTime()
+{
+    LogTime currentTime;
+
+#if (defined WIN32 || defined _WIN32)
+    SYSTEMTIME systemTime;
+    GetLocalTime(&systemTime);
+
+    char temp[8] = { 0 };
+    sprintf(temp, "%04d", systemTime.wYear);
+    currentTime.year=string(temp);
+    sprintf(temp, "%02d", systemTime.wMonth);
+    currentTime.month=string(temp);
+    sprintf(temp, "%02d", systemTime.wDay);
+    currentTime.day=string(temp);
+    sprintf(temp, "%02d", systemTime.wHour);
+    currentTime.hour=string(temp);
+    sprintf(temp, "%02d", systemTime.wMinute);
+    currentTime.minute=string(temp);
+    sprintf(temp, "%02d", systemTime.wSecond);
+    currentTime.second=string(temp);
+    sprintf(temp, "%03d", systemTime.wMilliseconds);
+    currentTime.millisecond=string(temp);
+    sprintf(temp, "%d", systemTime.wDayOfWeek);
+    currentTime.weekDay=string(temp);
+#else
+    struct timeval    tv;
+    struct tm         *p;
+    gettimeofday(&tv, NULL);
+    p = localtime(&tv.tv_sec);
+
+    char temp[8]={0};
+    sprintf(temp,"%04d",1900+p->tm_year);
+    currentTime.year=string(temp);
+    sprintf(temp,"%02d",1+p->tm_mon);
+    currentTime.month=string(temp);
+    sprintf(temp,"%02d",p->tm_mday);
+    currentTime.day=string(temp);
+    sprintf(temp,"%02d",p->tm_hour);
+    currentTime.hour=string(temp);
+    sprintf(temp,"%02d",p->tm_min);
+    currentTime.minute=string(temp);
+    sprintf(temp,"%02d",p->tm_sec);
+    currentTime.second=string(temp);
+    sprintf(temp,"%03d",(int)(tv.tv_usec/1000));
+    currentTime.millisecond = string(temp);
+    sprintf(temp, "%03d", (int)(tv.tv_usec % 1000));
+    currentTime.microsecond = string(temp);
+    sprintf(temp, "%d", p->tm_wday);
+    currentTime.weekDay = string(temp);
+#endif
+    return currentTime;
+}
+
+#define LOG_TIME(logFile) \
+do\
+{\
+    LogTime currentTime=GetTime(); \
+    fprintf(((logFile == NULL) ? stdout : logFile), "%s-%s-%s %s:%s:%s.%s\t",currentTime.year.c_str(),currentTime.month.c_str(),currentTime.day.c_str(),currentTime.hour.c_str(),currentTime.minute.c_str(),currentTime.second.c_str(),currentTime.millisecond.c_str()); \
+}while (0)
+
+
+#define LOG_INFO(logFile,logInfo, ...) \
+do\
+{\
+    LOCK; \
+    LOG_TIME(logFile); \
+    fprintf(((logFile == NULL) ? stdout : logFile), "INFO\t"); \
+    fprintf(((logFile == NULL) ? stdout : logFile), "[%s:%d (%s) ]: ", __FILE__, __LINE__, __FUNCTION__); \
+    fprintf(((logFile == NULL) ? stdout : logFile), logInfo, ## __VA_ARGS__); \
+    fflush(logFile); \
+    UNLOCK; \
+} while (0)
+
+#define LOG_DEBUG(logFile,logInfo, ...) \
+do\
+{\
+    LOCK; \
+    LOG_TIME(logFile);\
+    fprintf(((logFile==NULL)?stdout:logFile), "DEBUG\t"); \
+    fprintf(((logFile==NULL)?stdout:logFile), "[%s:%d (%s) ]: ", __FILE__, __LINE__, __FUNCTION__); \
+    fprintf(((logFile==NULL)?stdout:logFile),logInfo, ## __VA_ARGS__); \
+    fflush(logFile); \
+    UNLOCK; \
+} while (0)
+
+#define LOG_ERROR(logFile,logInfo, ...) \
+do\
+{\
+    LOCK; \
+    LOG_TIME(logFile);\
+    fprintf(((logFile==NULL)?stdout:logFile), "ERROR\t"); \
+    fprintf(((logFile==NULL)?stdout:logFile), "[%s:%d (%s) ]: ", __FILE__, __LINE__, __FUNCTION__); \
+    fprintf(((logFile==NULL)?stdout:logFile),logInfo, ## __VA_ARGS__); \
+    fflush(logFile); \
+    UNLOCK; \
+} while (0)
+
+#define LOG_WARN(logFile,logInfo, ...) \
+do\
+{\
+    LOCK; \
+    LOG_TIME(logFile);\
+    fprintf(((logFile==NULL)?stdout:logFile), "WARN\t"); \
+    fprintf(((logFile==NULL)?stdout:logFile), "[%s:%d (%s) ]: ", __FILE__, __LINE__, __FUNCTION__); \
+    fprintf(((logFile==NULL)?stdout:logFile),logInfo, ## __VA_ARGS__); \
+    fflush(logFile); \
+    UNLOCK; \
+} while (0)
+
+#endif // __SIMPLE_LOG_H__
+

Src/Utility/tokenization.cpp

+#include <stdexcept>
+#include <algorithm>
+#include <cstring>
+#include <fstream>
+#include "utf8proc.h"
+
+#include "./tokenization.h"
+
+
+namespace cuBERT {
+
+    void FullTokenizer::convert_tokens_to_ids(const std::vector<std::string> &tokens, uint64_t *ids) {
+        for (int i = 0; i < tokens.size(); ++i) {
+            ids[i] = convert_token_to_id(tokens[i]);
+        }
+    }
+
+// trim from start (in place)
+    static inline void ltrim(std::string &s) {
+        s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
+            return !std::isspace(ch);
+        }));
+    }
+
+// trim from end (in place)
+    static inline void rtrim(std::string &s) {
+        s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) {
+            return !std::isspace(ch);
+        }).base(), s.end());
+    }
+
+// trim from both ends (in place)
+    static inline void trim(std::string &s) {
+        ltrim(s);
+        rtrim(s);
+    }
+
+    void load_vocab(const char *vocab_file, std::unordered_map<std::string, uint64_t> *vocab) {
+        std::ifstream file(vocab_file);
+        if (!file) {
+            throw std::invalid_argument("Unable to open vocab file");
+        }
+
+        unsigned int index = 0;
+        std::string line;
+        while (std::getline(file, line)) {
+            trim(line);
+            (*vocab)[line] = index;
+            index++;
+        }
+
+        file.close();
+    }
+
+    inline bool _is_whitespace(int c, const char *cat) {
+        if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
+            return true;
+        }
+        return cat[0] == 'Z' && cat[1] == 's';
+    }
+
+    inline bool _is_control(int c, const char *cat) {
+        // These are technically control characters but we count them as whitespace characters.
+        if (c == '\t' || c == '\n' || c == '\r') {
+            return false;
+        }
+        return 'C' == *cat;
+    }
+
+    inline bool _is_punctuation(int cp, const char *cat) {
+// We treat all non-letter/number ASCII as punctuation.
+// Characters such as "^", "$", and "`" are not in the Unicode
+// Punctuation class but we treat them as punctuation anyways, for
+// consistency.
+        if ((cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) ||
+            (cp >= 91 && cp <= 96) || (cp >= 123 && cp <= 126)) {
+            return true;
+        }
+        return 'P' == *cat;
+    }
+
+    bool _is_whitespace(int c) {
+        return _is_whitespace(c, utf8proc_category_string(c));
+    }
+
+    bool _is_control(int c) {
+        return _is_control(c, utf8proc_category_string(c));
+    }
+
+    bool _is_punctuation(int cp) {
+        return _is_punctuation(cp, utf8proc_category_string(cp));
+    }
+
+    bool BasicTokenizer::_is_chinese_char(int cp) {
+// This defines a "chinese character" as anything in the CJK Unicode block:
+//   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+//
+// Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+// despite its name. The modern Korean Hangul alphabet is a different block,
+// as is Japanese Hiragana and Katakana. Those alphabets are used to write
+// space-separated words, so they are not treated specially and handled
+// like the all of the other languages.
+        return (cp >= 0x4E00 && cp <= 0x9FFF) ||
+               (cp >= 0x3400 && cp <= 0x4DBF) ||
+               (cp >= 0x20000 && cp <= 0x2A6DF) ||
+               (cp >= 0x2A700 && cp <= 0x2B73F) ||
+               (cp >= 0x2B740 && cp <= 0x2B81F) ||
+               (cp >= 0x2B820 && cp <= 0x2CEAF) ||
+               (cp >= 0xF900 && cp <= 0xFAFF) ||
+               (cp >= 0x2F800 && cp <= 0x2FA1F);
+    }
+
+    void BasicTokenizer::tokenize(const char *text, std::vector<std::string> *output_tokens, size_t max_length) {
+// This was added on November 1st, 2018 for the multilingual and Chinese
+// models. This is also applied to the English models now, but it doesn't
+// matter since the English models were not trained on any Chinese data
+// and generally don't have any Chinese data in them (there are Chinese
+// characters in the vocabulary because Wikipedia does have some Chinese
+// words in the English Wikipedia.).
+        if (do_lower_case) {
+            text = (const char *) utf8proc_NFD((const utf8proc_uint8_t *) text);
+        }
+
+        size_t word_bytes = std::strlen(text);
+        bool new_token = true;
+        size_t subpos = 0;
+        int cp;
+        char dst[4];
+
+        while (word_bytes > 0) {
+            int len = utf8proc_iterate((const utf8proc_uint8_t *) text + subpos, word_bytes, &cp);
+            if (len < 0) {
+                std::cerr << "UTF-8 decode error: " << text << std::endl;
+                break;
+            }
+            if (do_lower_case) {
+                cp = utf8proc_tolower(cp);
+            }
+
+            const char *cat = utf8proc_category_string(cp);
+            if (cp == 0 || cp == 0xfffd || _is_control(cp, cat)) {
+                // pass
+            } else if (do_lower_case && cat[0] == 'M' && cat[1] == 'n') {
+                // pass
+            } else if (_is_whitespace(cp, cat)) {
+                new_token = true;
+            } else {
+                size_t dst_len = len;
+                const char *dst_ptr = text + subpos;
+                if (do_lower_case) {
+                    dst_len = utf8proc_encode_char(cp, (utf8proc_uint8_t *) dst);
+                    dst_ptr = dst;
+                }
+
+                if (_is_punctuation(cp, cat) || _is_chinese_char(cp)) {
+                    output_tokens->emplace_back(dst_ptr, dst_len);
+                    new_token = true;
+                } else {
+                    if (new_token) {
+                        output_tokens->emplace_back(dst_ptr, dst_len);
+                        new_token = false;
+                    } else {
+                        output_tokens->at(output_tokens->size() - 1).append(dst_ptr, dst_len);
+                    }
+                }
+            }
+
+            word_bytes = word_bytes - len;
+            subpos = subpos + len;
+
+            // early terminate
+            if (output_tokens->size() >= max_length) {
+                break;
+            }
+        }
+
+        if (do_lower_case) {
+            free((void *) text);
+        }
+    }
+
+
+    void WordpieceTokenizer::tokenize(const std::string &token, std::vector<std::string> *output_tokens) {
+        if (token.size() > max_input_chars_per_word) {  // FIXME: slightly different
+            output_tokens->push_back(unk_token);
+            return;
+        }
+        size_t output_tokens_len = output_tokens->size();
+
+        for (size_t start = 0; start < token.size();) {
+            bool is_bad = true;
+
+            // TODO: can be optimized by prefix-tree
+            for (size_t end = token.size(); start < end; --end) {  // FIXME: slightly different
+                std::string substr = start > 0
+                                     ? "##" + token.substr(start, end - start)
+                                     : token.substr(start, end - start);
+                if (vocab->count(substr)) {
+                    is_bad = false;
+                    output_tokens->push_back(substr);
+                    start = end;
+                    break;
+                }
+            }
+
+            if (is_bad) {
+                output_tokens->resize(output_tokens_len);
+                output_tokens->push_back(unk_token);
+                return;
+            }
+        }
+    }
+
+
+    void FullTokenizer::tokenize(const char *text, std::vector<std::string> *output_tokens, size_t max_length) {
+        std::vector<std::string> tokens;
+        tokens.reserve(max_length);
+        basic_tokenizer->tokenize(text, &tokens, max_length);
+
+        for (const auto &token : tokens) {
+            wordpiece_tokenizer->tokenize(token, output_tokens);
+
+            // early terminate
+            if (output_tokens->size() >= max_length) {
+                break;
+            }
+        }
+    }
+}

Src/Utility/tokenization.h

+#ifndef CUBERT_TOKENIZATION_H
+#define CUBERT_TOKENIZATION_H
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <iostream>
+
+namespace cuBERT {
+
+    void load_vocab(const char *vocab_file, std::unordered_map<std::string, uint64_t> *vocab);
+
+/**
+ * Checks whether `chars` is a whitespace character.
+ * @param c
+ * @return
+ */
+    bool _is_whitespace(int c);
+
+/**
+ * Checks whether `chars` is a control character.
+ * @param c
+ * @return
+ */
+    bool _is_control(int c);
+
+/**
+ * Checks whether `chars` is a punctuation character.
+ * @param cp
+ * @return
+ */
+    bool _is_punctuation(int cp);
+
+/**
+ * Runs basic tokenization (punctuation splitting, lower casing, etc.).
+ */
+    class BasicTokenizer {
+    public:
+        /**
+         * Constructs a BasicTokenizer.
+         * @param do_lower_case Whether to lower case the input.
+         */
+        explicit BasicTokenizer(bool do_lower_case = true) : do_lower_case(do_lower_case) {}
+
+        BasicTokenizer(const BasicTokenizer &other) = delete;
+
+        virtual ~BasicTokenizer() = default;
+
+        /**
+         * Tokenizes a piece of text.
+         *
+         * to_lower
+         * _run_strip_accents Strips accents from a piece of text.
+         * _clean_text Performs invalid character removal and whitespace cleanup on text.
+         * _tokenize_chinese_chars Adds whitespace around any CJK character.
+         * _run_split_on_punc Splits punctuation on a piece of text.
+         * whitespace_tokenize Runs basic whitespace cleaning and splitting on a piece of text.
+         *
+         * @param text
+         * @param output_tokens
+         */
+        void tokenize(const char *text, std::vector<std::string> *output_tokens, size_t max_length);
+
+    private:
+        const bool do_lower_case;
+
+        /**
+         * Checks whether CP is the codepoint of a CJK character.
+         * @param cp
+         * @return
+         */
+        inline static bool _is_chinese_char(int cp);
+    };
+
+/**
+ * Runs WordPiece tokenziation.
+ */
+    class WordpieceTokenizer {
+    public:
+        explicit WordpieceTokenizer(
+                std::unordered_map<std::string, uint64_t> *vocab,
+                std::string unk_token = "[UNK]",
+                int max_input_chars_per_word = 200
+        ) : vocab(vocab), unk_token(unk_token), max_input_chars_per_word(max_input_chars_per_word) {}
+
+        WordpieceTokenizer(const WordpieceTokenizer &other) = delete;
+
+        virtual ~WordpieceTokenizer() = default;
+
+        /**
+         * Tokenizes a piece of text into its word pieces.
+         *
+         * This uses a greedy longest-match-first algorithm to perform tokenization
+         * using the given vocabulary.
+         *
+         * For example:
+         *   input = "unaffable"
+         *   output = ["un", "##aff", "##able"]
+         *
+         * @param text A single token or whitespace separated tokens. This should have already been passed through `BasicTokenizer.
+         * @param output_tokens A list of wordpiece tokens.
+         */
+        void tokenize(const std::string &text, std::vector<std::string> *output_tokens);
+
+    private:
+        const std::unordered_map<std::string, uint64_t> *vocab;
+        const std::string unk_token;
+        const int max_input_chars_per_word;
+    };
+
+
+/**
+ * Runs end-to-end tokenziation.
+ */
+    class FullTokenizer {
+    public:
+        FullTokenizer(const char *vocab_file, bool do_lower_case = true) {
+            vocab = new std::unordered_map<std::string, uint64_t>();
+            load_vocab(vocab_file, vocab);
+            basic_tokenizer = new BasicTokenizer(do_lower_case);
+            wordpiece_tokenizer = new WordpieceTokenizer(vocab);
+        }
+
+        ~FullTokenizer() {
+            if (wordpiece_tokenizer != NULL){
+                wordpiece_tokenizer = NULL;
+            }
+            delete wordpiece_tokenizer;
+
+            if (basic_tokenizer != NULL){
+                basic_tokenizer = NULL;
+            }
+            delete basic_tokenizer;
+
+            if (vocab != NULL){
+                vocab = NULL;
+            }
+            delete vocab;
+        }
+
+        void tokenize(const char *text, std::vector<std::string> *output_tokens, size_t max_length);
+
+        inline uint64_t convert_token_to_id(const std::string &token) {
+            auto item = vocab->find(token);
+            if (item == vocab->end()) {
+                std::cerr << "vocab missing key: " << token << std::endl;
+                return 0;
+            } else {
+                return item->second;
+            }
+        }
+
+        void convert_tokens_to_ids(const std::vector<std::string> &tokens, uint64_t *ids);
+
+    private:
+        std::unordered_map<std::string, uint64_t> *vocab;
+        BasicTokenizer *basic_tokenizer;
+        WordpieceTokenizer *wordpiece_tokenizer;
+    };
+
+}
+
+#endif //CUBERT_TOKENIZATION_H

Src/main.cpp

+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <Bert.h>
+#include <SimpleLog.h>
+#include <Filesystem.h>
+#include <tokenization.h>
+
+int main(int argc, char *argv[])
+{
+    // 加载Bert模型
+    ortSamples::Bert bert;
+    ortSamples::ErrorCode errorCode = bert.Initialize();
+    if (errorCode != ortSamples::SUCCESS)
+    {
+        LOG_ERROR(stdout, "fail to initialize Bert!\n");
+        exit(-1);
+    }
+    LOG_INFO(stdout, "succeed to initialize Bert\n");
+
+    int max_seq_length = 256;      // 滑动窗口的长度
+    int max_query_length = 64;     // 问题的最大长度
+    int batch_size = 1;            // batch_size值
+    int n_best_size = 20;          // 索引数量
+    int max_answer_length = 30;    // 答案的最大长度
+
+    // 上下文文本数据
+    const char text[] = { u8"ROCm is the first open-source exascale-class platform for accelerated computing that’s also programming-language independent. It brings a philosophy of choice, minimalism and modular software development to GPU computing. You are free to choose or even develop tools and a language run time for your application. ROCm is built for scale, it supports multi-GPU computing and has a rich system run time with the critical features that large-scale application, compiler and language-run-time development requires. Since the ROCm ecosystem is comprised of open technologies: frameworks (Tensorflow / PyTorch), libraries (MIOpen / Blas / RCCL), programming model (HIP), inter-connect (OCD) and up streamed Linux® Kernel support – the platform is continually optimized for performance and extensibility." };
+    char question[100];
+
+    std::vector<std::vector<long unsigned int>> input_ids;
+    std::vector<std::vector<long unsigned int>> input_masks;
+    std::vector<std::vector<long unsigned int>> segment_ids;
+    std::vector<float> start_position;
+    std::vector<float> end_position;
+    std::string answer = {};
+
+    cuBERT::FullTokenizer tokenizer = cuBERT::FullTokenizer("../Resource/uncased_L-12_H-768_A-12/vocab.txt");      // 分词工具
+
+    while (true) 
+    {
+        // 数据前处理
+        std::cout << "question: ";
+        cin.getline(question, 100);
+        bert.Preprocessing(tokenizer, batch_size, max_seq_length, text, question, input_ids, input_masks, segment_ids); 
+
+        // 推理
+        bert.Inference(input_ids, input_masks, segment_ids, start_position, end_position);
+
+        // 数据后处理
+        bert.Postprocessing(n_best_size, max_answer_length, start_position, end_position, answer);
+
+        // 打印输出预测结果
+        std::cout << "answer: " << answer << std::endl;
+
+        // 清除数据
+        input_ids.clear();
+        input_masks.clear();
+        segment_ids.clear();
+        start_position.clear();
+        end_position.clear();
+        answer = {};
+    }
+    
+    return 0;
+}
\ No newline at end of file

model.properties

+# 模型名称
+modelName=Bert_ONNXRuntime
+# 模型描述
+modelDescription=Bert是一种预训练的语言表征模型。
+# 应用场景
+appScenario=推理,NLP,ONNXRuntime
+# 框架类型
+frameType=ONNXRuntime
\ No newline at end of file