feat(sfcs): download libcfs.so at runtime

Signed-off-by: huteng.ht <huteng.ht@bytedance.com>

setup.py

@@ -110,6 +110,7 @@ setup(
         "numpy",
         "loguru",
         "requests-unixsocket",
+        "requests",
     ],
     include_package_data=True,
     cmdclass={"build_ext": BuildExtension},

veturboio/ops/load_utils.py

@@ -22,8 +22,9 @@ from loguru import logger
 from veturboio.ops.cipher import CipherInfo
 
 try:
-    import veturboio_ext
+    from veturboio.utils.load_veturboio_ext import load_veturboio_ext
 
+    veturboio_ext = load_veturboio_ext()
     IOHelper = veturboio_ext.IOHelper
 except ImportError:
     IOHelper = None

veturboio/ops/sfcs_utils.py

@@ -28,8 +28,9 @@ from loguru import logger
 from veturboio.ops.cipher import CipherInfo, DataPipeClient
 
 try:
-    import veturboio_ext
+    from veturboio.utils.load_veturboio_ext import load_veturboio_ext
 
+    veturboio_ext = load_veturboio_ext()
     SFCSFile = veturboio_ext.SFCSFile
 except ImportError:
     SFCSFile = None

veturboio/utils/load_veturboio_ext.py

+'''
+Copyright (c) 2024 Beijing Volcano Engine Technology Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+
+from loguru import logger
+
+LIBCFS_DEFAULT_URL = "https://veturbo-cn-beijing.tos-cn-beijing.volces.com/veturboio/libcfs/libcfs.so"
+LIBCFS_DEFAULT_PATH = "/usr/lib/libcfs.so"
+
+
+def load_libcfs():
+    libcfs_path = os.getenv("LIBCFS_PATH", LIBCFS_DEFAULT_PATH)
+    if not os.path.isfile(libcfs_path):
+        # libcfs_path not exist, download from url
+        import requests
+
+        libcfs_url = os.getenv("LIBCFS_URL", LIBCFS_DEFAULT_URL)
+        logger.info(f"download libcfs.so from {libcfs_url}, save to {libcfs_path}")
+        r = requests.get(libcfs_url, timeout=60)
+        with open(libcfs_path, 'wb') as f:
+            f.write(r.content)
+
+
+def load_veturboio_ext():
+    load_libcfs()
+    import veturboio_ext
+
+    return veturboio_ext