Back out "Add example of how to build, link, and test an external SWI…

…G module" (facebookresearch#3954)

Summary:
Pull Request resolved: facebookresearch#3954

Original commit changeset: 0c1cd25eabbf

Original Phabricator Diff: D63991471

Reviewed By: mengdilin, asadoughi

Differential Revision: D64439886

fbshipit-source-id: cc90958f6d90a429a4eece8e1cd1322b20d9aceb

conda/faiss-gpu-raft/meta.yaml

@@ -112,7 +112,7 @@ outputs:
         - pytorch
         - pytorch-cuda {{ cuda_constraints }}
       commands:
-        - python -X faulthandler -m unittest discover -v -s tests/ -p "(?!.*test_external_module\.py)test_.*py"
+        - python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
         - python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
         - cp tests/common_faiss_tests.py faiss/gpu/test
         - python -X faulthandler -m unittest discover -v -s faiss/gpu/test/ -p "test_*"

conda/faiss-gpu/meta.yaml

@@ -101,7 +101,7 @@ outputs:
         - pytorch
         - pytorch-cuda {{ cuda_constraints }}
       commands:
-        - python -X faulthandler -m unittest discover -v -s tests/ -p "(?!.*test_external_module\.py)test_.*py"
+        - python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
         - python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
         - cp tests/common_faiss_tests.py faiss/gpu/test
         - python -X faulthandler -m unittest discover -v -s faiss/gpu/test/ -p "test_*"

conda/faiss/meta.yaml

@@ -86,7 +86,7 @@ outputs:
         - scipy
         - pytorch
       commands:
-        - python -X faulthandler -m unittest discover -v -s tests/ -p "(?!.*test_external_module\.py)test_.*py"
+        - python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
         - python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
         - sh test_cpu_dispatch.sh  # [linux64]
       files:

faiss/python/CMakeLists.txt

@@ -61,7 +61,6 @@ configure_swigfaiss(swigfaiss.swig)
 configure_swigfaiss(swigfaiss_avx2.swig)
 configure_swigfaiss(swigfaiss_avx512.swig)
 configure_swigfaiss(swigfaiss_sve.swig)
-configure_swigfaiss(faiss_example_external_module.swig)
 
 if(TARGET faiss)
   # Manually add headers as extra dependencies of swigfaiss.
@@ -75,8 +74,6 @@ if(TARGET faiss)
       "${faiss_SOURCE_DIR}/faiss/${h}")
     list(APPEND SWIG_MODULE_swigfaiss_sve_EXTRA_DEPS
       "${faiss_SOURCE_DIR}/faiss/${h}")
-    list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
-      "${faiss_SOURCE_DIR}/faiss/${h}")
   endforeach()
   if(FAISS_ENABLE_ROCM)
     foreach(h ${FAISS_GPU_HEADERS})
@@ -86,8 +83,6 @@ if(TARGET faiss)
         "${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
       list(APPEND SWIG_MODULE_swigfaiss_avx512_EXTRA_DEPS
         "${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
-      list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
-        "${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
     endforeach()
   else()
     foreach(h ${FAISS_GPU_HEADERS})
@@ -99,8 +94,6 @@ if(TARGET faiss)
         "${faiss_SOURCE_DIR}/faiss/gpu/${h}")
       list(APPEND SWIG_MODULE_swigfaiss_sve_EXTRA_DEPS
         "${faiss_SOURCE_DIR}/faiss/gpu/${h}")
-      list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
-        "${faiss_SOURCE_DIR}/faiss/gpu/${h}")
     endforeach()
   endif()
 else()
@@ -159,37 +152,25 @@ if(NOT FAISS_OPT_LEVEL STREQUAL "sve")
   set_target_properties(swigfaiss_sve PROPERTIES EXCLUDE_FROM_ALL TRUE)
 endif()
 
-set_property(SOURCE faiss_example_external_module.swig
-  PROPERTY SWIG_MODULE_NAME faiss_example_external_module)
-swig_add_library(faiss_example_external_module
-  TYPE SHARED
-  LANGUAGE python
-  SOURCES faiss_example_external_module.swig
-)
-set_property(TARGET faiss_example_external_module PROPERTY SWIG_COMPILE_OPTIONS -doxygen)
-
 if(NOT WIN32)
   # NOTE: Python does not recognize the dylib extension.
   set_target_properties(swigfaiss PROPERTIES SUFFIX .so)
   set_target_properties(swigfaiss_avx2 PROPERTIES SUFFIX .so)
   set_target_properties(swigfaiss_avx512 PROPERTIES SUFFIX .so)
   set_target_properties(swigfaiss_sve PROPERTIES SUFFIX .so)
-  set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .so)
 else()
   # we need bigobj for the swig wrapper
   target_compile_options(swigfaiss PRIVATE /bigobj)
   target_compile_options(swigfaiss_avx2 PRIVATE /bigobj)
   target_compile_options(swigfaiss_avx512 PRIVATE /bigobj)
   target_compile_options(swigfaiss_sve PRIVATE /bigobj)
-  target_compile_options(faiss_example_external_module PRIVATE /bigobj)
 endif()
 
 if(FAISS_ENABLE_GPU)
   if(FAISS_ENABLE_ROCM)
     target_link_libraries(swigfaiss PRIVATE hip::host)
     target_link_libraries(swigfaiss_avx2 PRIVATE hip::host)
     target_link_libraries(swigfaiss_avx512 PRIVATE hip::host)
-    target_link_libraries(faiss_example_external_module PRIVATE hip::host)
   else()
     find_package(CUDAToolkit REQUIRED)
     if(FAISS_ENABLE_RAFT)
@@ -240,21 +221,12 @@ target_link_libraries(swigfaiss_sve PRIVATE
   OpenMP::OpenMP_CXX
 )
 
-target_link_libraries(faiss_example_external_module PRIVATE
-  Python::Module
-  Python::NumPy
-  OpenMP::OpenMP_CXX
-  swigfaiss
-  faiss
-)
-
 # Hack so that python_callbacks.h can be included as
 # `#include <faiss/python/python_callbacks.h>`.
 target_include_directories(swigfaiss PRIVATE ${PROJECT_SOURCE_DIR}/../..)
 target_include_directories(swigfaiss_avx2 PRIVATE ${PROJECT_SOURCE_DIR}/../..)
 target_include_directories(swigfaiss_avx512 PRIVATE ${PROJECT_SOURCE_DIR}/../..)
 target_include_directories(swigfaiss_sve PRIVATE ${PROJECT_SOURCE_DIR}/../..)
-target_include_directories(faiss_example_external_module PRIVATE ${PROJECT_SOURCE_DIR}/../..)
 
 find_package(Python REQUIRED
   COMPONENTS Development NumPy
@@ -280,7 +252,6 @@ target_link_libraries(swigfaiss PRIVATE faiss_python_callbacks)
 target_link_libraries(swigfaiss_avx2 PRIVATE faiss_python_callbacks)
 target_link_libraries(swigfaiss_avx512 PRIVATE faiss_python_callbacks)
 target_link_libraries(swigfaiss_sve PRIVATE faiss_python_callbacks)
-target_link_libraries(faiss_example_external_module PRIVATE faiss_python_callbacks)
 
 configure_file(setup.py setup.py COPYONLY)
 configure_file(__init__.py __init__.py COPYONLY)

faiss/python/setup.py

@@ -4,12 +4,10 @@
 # LICENSE file in the root directory of this source tree.
 
 from __future__ import print_function
-
+from setuptools import setup, find_packages
 import os
-import platform
 import shutil
-
-from setuptools import find_packages, setup
+import platform
 
 # make the faiss python package dir
 shutil.rmtree("faiss", ignore_errors=True)
@@ -22,32 +20,25 @@
 shutil.copyfile("extra_wrappers.py", "faiss/extra_wrappers.py")
 shutil.copyfile("array_conversions.py", "faiss/array_conversions.py")
 
-ext = ".pyd" if platform.system() == "Windows" else ".so"
-prefix = "Release/" * (platform.system() == "Windows")
+ext = ".pyd" if platform.system() == 'Windows' else ".so"
+prefix = "Release/" * (platform.system() == 'Windows')
 
 swigfaiss_generic_lib = f"{prefix}_swigfaiss{ext}"
 swigfaiss_avx2_lib = f"{prefix}_swigfaiss_avx2{ext}"
 swigfaiss_avx512_lib = f"{prefix}_swigfaiss_avx512{ext}"
 callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}"
 swigfaiss_sve_lib = f"{prefix}_swigfaiss_sve{ext}"
-faiss_example_external_module_lib = f"_faiss_example_external_module{ext}"
 
 found_swigfaiss_generic = os.path.exists(swigfaiss_generic_lib)
 found_swigfaiss_avx2 = os.path.exists(swigfaiss_avx2_lib)
 found_swigfaiss_avx512 = os.path.exists(swigfaiss_avx512_lib)
 found_callbacks = os.path.exists(callbacks_lib)
 found_swigfaiss_sve = os.path.exists(swigfaiss_sve_lib)
-found_faiss_example_external_module_lib = os.path.exists(
-    faiss_example_external_module_lib
-)
 
-assert (
-    found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve or found_faiss_example_external_module_lib
-), (
-    f"Could not find {swigfaiss_generic_lib} or "
-    f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib} or {faiss_example_external_module_lib}. "
+assert (found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve), \
+    f"Could not find {swigfaiss_generic_lib} or " \
+    f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib}. " \
     f"Faiss may not be compiled yet."
-)
 
 if found_swigfaiss_generic:
     print(f"Copying {swigfaiss_generic_lib}")
@@ -73,17 +64,7 @@
     shutil.copyfile("swigfaiss_sve.py", "faiss/swigfaiss_sve.py")
     shutil.copyfile(swigfaiss_sve_lib, f"faiss/_swigfaiss_sve{ext}")
 
-if found_faiss_example_external_module_lib:
-    print(f"Copying {faiss_example_external_module_lib}")
-    shutil.copyfile(
-        "faiss_example_external_module.py", "faiss/faiss_example_external_module.py"
-    )
-    shutil.copyfile(
-        faiss_example_external_module_lib,
-        f"faiss/_faiss_example_external_module{ext}",
-    )
-
-long_description = """
+long_description="""
 Faiss is a library for efficient similarity search and clustering of dense
 vectors. It contains algorithms that search in sets of vectors of any size,
  up to ones that possibly do not fit in RAM. It also contains supporting
@@ -92,19 +73,20 @@
 are implemented on the GPU. It is developed by Facebook AI Research.
 """
 setup(
-    name="faiss",
-    version="1.9.0",
-    description="A library for efficient similarity search and clustering of dense vectors",
+    name='faiss',
+    version='1.9.0',
+    description='A library for efficient similarity search and clustering of dense vectors',
     long_description=long_description,
-    url="https://github.com/facebookresearch/faiss",
-    author="Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini",
-    author_email="matthijs@meta.com",
-    license="MIT",
-    keywords="search nearest neighbors",
-    install_requires=["numpy", "packaging"],
-    packages=["faiss", "faiss.contrib", "faiss.contrib.torch"],
+    url='https://github.com/facebookresearch/faiss',
+    author='Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini',
+    author_email='matthijs@meta.com',
+    license='MIT',
+    keywords='search nearest neighbors',
+
+    install_requires=['numpy', 'packaging'],
+    packages=['faiss', 'faiss.contrib', 'faiss.contrib.torch'],
     package_data={
-        "faiss": ["*.so", "*.pyd"],
+        'faiss': ['*.so', '*.pyd'],
     },
     zip_safe=False,
 )