diff --git a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt
index 8783d2b065d8..9e2e7720a94a 100644
--- a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt
+++ b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt
@@ -110,6 +110,25 @@ foreach(LIB_TARGET ${ARROW_FLIGHT_SQL_ODBC_LIBRARIES})
   target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_FLIGHT_SQL_ODBC_EXPORTING)
 endforeach()
 
+if(NOT WIN32)
+  if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
+    set(ARROW_FLIGHT_SQL_ODBC_SO "${CMAKE_INSTALL_LIBDIR}")
+  else()
+    set(ARROW_FLIGHT_SQL_ODBC_SO "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+  endif()
+  string(APPEND ARROW_FLIGHT_SQL_ODBC_SO
+         "/$<TARGET_FILE_NAME:arrow_flight_sql_odbc_shared>")
+  # Expand only @...@. $<...> aren't expanded.
+  configure_file("arrow-flight-sql-odbc-template.ini.in"
+                 "arrow-flight-sql-odbc-template.ini.generate.in" @ONLY)
+  # Expand only $<...>.
+  file(GENERATE
+       OUTPUT "$<CONFIG>/arrow-flight-sql-odbc-template.ini"
+       INPUT "${CMAKE_CURRENT_BINARY_DIR}/arrow-flight-sql-odbc-template.ini.generate.in")
+  install(FILES "${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/arrow-flight-sql-odbc-template.ini"
+          DESTINATION "${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/flight/sql/odbc")
+endif()
+
 # Construct ODBC Windows installer. Only Release installer is supported
 if(ARROW_FLIGHT_SQL_ODBC_INSTALLER)
 
diff --git a/cpp/src/arrow/flight/sql/odbc/arrow-flight-sql-odbc-template.ini.in b/cpp/src/arrow/flight/sql/odbc/arrow-flight-sql-odbc-template.ini.in
new file mode 100644
index 000000000000..9f0efe0d0d4e
--- /dev/null
+++ b/cpp/src/arrow/flight/sql/odbc/arrow-flight-sql-odbc-template.ini.in
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[Apache Arrow Flight SQL ODBC Driver]
+Description=An ODBC Driver for Apache Arrow Flight SQL
+Driver=@ARROW_FLIGHT_SQL_ODBC_SO@
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
index 016954db0511..c8827428c0c3 100755
--- a/dev/release/verify-yum.sh
+++ b/dev/release/verify-yum.sh
@@ -261,6 +261,7 @@ if [ "${have_flight}" = "yes" ]; then
 
   echo "::group::Test Apache Arrow Flight SQL ODBC driver"
   ${install_command} ${enablerepo_epel} arrow-flight-sql-odbc-devel-${package_version}
+  odbcinst -q -d -n 'Apache Arrow Flight SQL ODBC Driver'
   echo "::endgroup::"
 fi
 
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 3b2964cc68ca..aca2d468332d 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -494,14 +494,23 @@ Libraries and header files for Apache Arrow Flight SQL.
 Summary:	C++ library for interacting with SQL databases via ODBC.
 License:	Apache-2.0
 Requires:	%{name}%{so_version}-flight-sql-libs = %{version}-%{release}
+Requires(post):	/usr/bin/odbcinst
+Requires(postun):	/usr/bin/odbcinst
 
 %description -n %{name}%{so_version}-flight-sql-odbc-libs
 This package contains the libraries for Apache Arrow Flight SQL ODBC driver.
 
+%post -n %{name}%{so_version}-flight-sql-odbc-libs
+odbcinst -i -d -f %{_datadir}/arrow/flight/sql/odbc/arrow-flight-sql-odbc-template.ini
+
+%postun -n %{name}%{so_version}-flight-sql-odbc-libs
+odbcinst -u -d -n "Apache Arrow Flight SQL ODBC Driver"
+
 %files -n %{name}%{so_version}-flight-sql-odbc-libs
 %defattr(-,root,root,-)
 %doc README.md
 %license LICENSE.txt NOTICE.txt
+%{_datadir}/arrow/flight/sql/odbc/arrow-flight-sql-odbc-template.ini
 %{_libdir}/libarrow_flight_sql_odbc.so.*
 
 %package flight-sql-odbc-devel