Linking to stdc++fs (std::experimental::filesystem) doesn't work

This is my CMakeLists.txt file for a project I am trying to run, everything seems fine up until the filesystem library, I tried a couple of things and what worked for me is the std::experimental::filesystem

g++ and gcc version g++ (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0, gcc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0

cmake_minimum_required(VERSION 2.8.3)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
project(emsgc1)

find_package(catkin_simple REQUIRED)
catkin_simple(ALL_DEPS_REQUIRED)

SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3")

find_package(OpenCV REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_check_modules(gsl REQUIRED gsl)

include_directories("/usr/include/eigen3")
###########
## Build ##
###########

## Specify additional locations of header files
## Your package locations should be listed before other locations
include_directories(include
    third-party/gco
    third-party/delaunator/include
    ${OpenCV_INCLUDE_DIRS})

set(HEADERS
    include/emsgc/core/event_motion_segmentation.h
    include/emsgc/core/image_warped_events.h
    include/emsgc/core/numerical_deriv.h
    include/emsgc/core/contrastFunctor.h
    include/emsgc/container/EventQueueMat.h
    include/emsgc/container/EventMRF.h
    include/emsgc/container/PerspectiveCamera.h
    include/emsgc/tools/utils.h
    include/emsgc/tools/TicToc.h
    third-party/gco/energy.h
    third-party/gco/block.h
    third-party/gco/GCoptimization.h
    third-party/gco/graph.h
    third-party/gco/LinkedBlockList.h
    third-party/delaunator/include/delaunator.hpp)

set(HEADERS
    include/emsgc/core/event_motion_segmentation.h
    include/emsgc/core/image_warped_events.h
    include/emsgc/core/numerical_deriv.h
    include/emsgc/core/contrastFunctor.h
    include/emsgc/container/EventQueueMat.h
    include/emsgc/container/EventMRF.h
    include/emsgc/container/PerspectiveCamera.h
    include/emsgc/tools/utils.h
    include/emsgc/tools/TicToc.h
    third-party/gco/energy.h
    third-party/gco/block.h
    third-party/gco/GCoptimization.h
    third-party/gco/graph.h
    third-party/gco/LinkedBlockList.h
    third-party/delaunator/include/delaunator.hpp)

set(SOURCES
    src/container/PerspectiveCamera.cpp
    src/core/event_motion_segmentation.cpp
    src/core/image_warped_events.cpp
    src/core/optim_contrast_gsl.cpp
    src/core/numerical_deriv.cpp
    src/core/optimizeLabels.cpp
    src/core/optimizeModels.cpp
    third-party/gco/GCoptimization.cpp
    third-party/gco/graph.cpp
    third-party/gco/LinkedBlockList.cpp
    third-party/gco/maxflow.cpp)
cs_add_library(${PROJECT_NAME}_LIB ${HEADERS} ${SOURCES})

## Node
link_libraries(stdc++fs)
cs_add_executable(emsgc_main emsgc_main.cpp)
target_link_libraries(emsgc_main ${PROJECT_NAME}_LIB ${catkin_LIBRARIES} ${OpenCV_LIBRARIES} yaml-cpp ${gsl_LIBRARIES} "stdc++fs")

The error is that for some reason it is not linking the stdc++fs when it is there ate the end, if I try to post the verbose output, it flags as the question is mostly code haha

Errors     << emsgc:make /home/suhail/catkin_ws/logs/emsgc/build.make.002.log                                                                                                                              
CMakeFiles/emsgc_main.dir/emsgc_main.cpp.o: In function `std::experimental::filesystem::v1::__cxx11::path::path<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::experimental::filesystem::v1::__cxx11::path>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)':
emsgc_main.cpp:(.text._ZNSt12experimental10filesystem2v17__cxx114pathC2INSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES3_EERKT_[_ZNSt12experimental10filesystem2v17__cxx114pathC5INSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES3_EERKT_]+0x6c): undefined reference to `std::experimental::filesystem::v1::__cxx11::path::_M_split_cmpts()'
CMakeFiles/emsgc_main.dir/emsgc_main.cpp.o: In function `main':
emsgc_main.cpp:(.text.startup+0x8e7): undefined reference to `std::experimental::filesystem::v1::create_directories(std::experimental::filesystem::v1::__cxx11::path const&)'
emsgc_main.cpp:(.text.startup+0x929): undefined reference to `std::experimental::filesystem::v1::create_directories(std::experimental::filesystem::v1::__cxx11::path const&)'
emsgc_main.cpp:(.text.startup+0x969): undefined reference to `std::experimental::filesystem::v1::create_directories(std::experimental::filesystem::v1::__cxx11::path const&)'
collect2: error: ld returned 1 exit status
make[2]: *** [/home/suhail/catkin_ws/devel/lib/emsgc/emsgc_main] Error 1
make[1]: *** [CMakeFiles/emsgc_main.dir/all] Error 2
make: *** [all] Error 2

When I create a test.cpp file to try it out, it works

#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;

int main() {
    fs::path p = "/tmp";
    return 0;
}

using

g++ -std=c++17 test.cpp -lstdc++fs