[YSQL][Unit Test] heap-use-after-free in QLTabletTest.VerifyIndexRangeWithInconsistentTable

Jira Link: DB-5140

Description

Seen in 2.17.2.0-b51 Part of failure:

==================
WARNING: ThreadSanitizer: heap-use-after-free (pid=14530)
  Read of size 4 at 0x7b4800107e10 by thread T154:
    #0 yb::Schema::colocation_id() const ${BUILD_ROOT}/../../src/yb/common/schema.h:781:12 (libyb_docdb_encoding.so+0x3a235)
    #1 yb::docdb::DocKeyBelongsTo(yb::Slice, yb::Schema const&) ${BUILD_ROOT}/../../src/yb/docdb/doc_key.cc:1322:45 (libyb_docdb_encoding.so+0x3a235)
    #2 yb::docdb::DocRowwiseIterator::HasNext() ${BUILD_ROOT}/../../src/yb/docdb/doc_rowwise_iterator.cc:321:10 (libyb_docdb.so+0x1f8ec7)
    #3 yb::tablet::Tablet::VerifyTableConsistencyForCQL(std::vector<string, std::allocator<string>> const&, std::vector<yb::ColumnSchema, std::allocator<yb::ColumnSchema>> const&, string const&, int, std::chrono::time_point<yb::CoarseMonoClock, std::chrono::duration<long long, std::ratio<1l, 1000000000l>>>, yb::HybridTime, bool, std::unordered_map<string, unsigned long, std::hash<string>, std::equal_to<string>, std::allocator<std::pair<string const, unsigned long>>>*, string*) ${BUILD_ROOT}/../../src/yb/tablet/tablet.cc:2755:10 (libtablet.so+0x1870de)
    #4 yb::tablet::Tablet::VerifyIndexTableConsistencyForCQL(std::vector<yb::IndexInfo, std::allocator<yb::IndexInfo>> const&, string const&, int, std::chrono::time_point<yb::CoarseMonoClock, std::chrono::duration<long long, std::ratio<1l, 1000000000l>>>, yb::HybridTime, std::unordered_map<string, unsigned long, std::hash<string>, std::equal_to<string>, std::allocator<std::pair<string const, unsigned long>>>*, string*) ${BUILD_ROOT}/../../src/yb/tablet/tablet.cc:2701:10 (libtablet.so+0x186782)
    #5 yb::tserver::TabletServiceImpl::VerifyTableRowRange(yb::tserver::VerifyTableRowRangeRequestPB const*, yb::tserver::VerifyTableRowRangeResponsePB*, yb::rpc::RpcContext) ${BUILD_ROOT}/../../src/yb/tserver/tablet_service.cc:1001:36 (libtserver.so+0x226a72)
    #6 yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2::operator()(std::shared_ptr<yb::rpc::InboundCall>) const::'lambda'(yb::tserver::VerifyTableRowRangeRequestPB const*, yb::tserver::VerifyTableRowRangeResponsePB*, yb::rpc::RpcContext)::operator()(yb::tserver::VerifyTableRowRangeRequestPB const*, yb::tserver::VerifyTableRowRangeResponsePB*, yb::rpc::RpcContext) const ${BUILD_ROOT}/src/yb/tserver/tserver_service.service.cc:578:9 (libtserver_service_proto.so+0xb9907)
    #7 auto yb::rpc::HandleCall<yb::rpc::RpcCallPBParamsImpl<yb::tserver::VerifyTableRowRangeRequestPB, yb::tserver::VerifyTableRowRangeResponsePB>, yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2::operator()(std::shared_ptr<yb::rpc::InboundCall>) const::'lambda'(yb::tserver::VerifyTableRowRangeRequestPB const*, yb::tserver::VerifyTableRowRangeResponsePB*, yb::rpc::RpcContext)>(std::shared_ptr<yb::rpc::InboundCall>, yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2::operator()(std::shared_ptr<yb::rpc::InboundCall>) const::'lambda'(yb::tserver::VerifyTableRowRangeRequestPB const*, yb::tserver::VerifyTableRowRangeResponsePB*, yb::rpc::RpcContext)) ${BUILD_ROOT}/../../src/yb/rpc/local_call.h:115:7 (libtserver_service_proto.so+0xb9907)
    #8 yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2::operator()(std::shared_ptr<yb::rpc::InboundCall>) const ${BUILD_ROOT}/src/yb/tserver/tserver_service.service.cc:576:7 (libtserver_service_proto.so+0xb9907)
    #9 decltype(std::declval<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2&>()(std::declval<std::shared_ptr<yb::rpc::InboundCall>>())) std::__invoke[abi:v15003]<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2&, std::shared_ptr<yb::rpc::InboundCall>>(yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2&, std::shared_ptr<yb::rpc::InboundCall>&&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/invoke.h:394:23 (libtserver_service_proto.so+0xb9907)
    #10 void std::__invoke_void_return_wrapper<void, true>::__call<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2&, std::shared_ptr<yb::rpc::InboundCall>>(yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2&, std::shared_ptr<yb::rpc::InboundCall>&&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/invoke.h:479:9 (libtserver_service_proto.so+0xb9907)
    #11 std::__function::__alloc_func<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2, std::allocator<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2>, void (std::shared_ptr<yb::rpc::InboundCall>)>::operator()[abi:v15003](std::shared_ptr<yb::rpc::InboundCall>&&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:185:16 (libtserver_service_proto.so+0xb9907)
    #12 std::__function::__func<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2, std::allocator<yb::tserver::TabletServerServiceIf::InitMethods(scoped_refptr<yb::MetricEntity> const&)::$_2>, void (std::shared_ptr<yb::rpc::InboundCall>)>::operator()(std::shared_ptr<yb::rpc::InboundCall>&&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:359:12 (libtserver_service_proto.so+0xb9907)
    #13 std::__function::__value_func<void (std::shared_ptr<yb::rpc::InboundCall>)>::operator()[abi:v15003](std::shared_ptr<yb::rpc::InboundCall>&&) const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:512:16 (libtserver_service_proto.so+0xb7cc2)
    #14 std::function<void (std::shared_ptr<yb::rpc::InboundCall>)>::operator()(std::shared_ptr<yb::rpc::InboundCall>) const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:1197:12 (libtserver_service_proto.so+0xb7cc2)
    #15 yb::tserver::TabletServerServiceIf::Handle(std::shared_ptr<yb::rpc::InboundCall>) ${BUILD_ROOT}/src/yb/tserver/tserver_service.service.cc:531:3 (libtserver_service_proto.so+0xb7cc2)
    #16 yb::rpc::ServicePoolImpl::Handle(std::shared_ptr<yb::rpc::InboundCall>) ${BUILD_ROOT}/../../src/yb/rpc/service_pool.cc:263:19 (libyrpc.so+0x17ab7a)
    #17 yb::rpc::InboundCall::InboundCallTask::Run() ${BUILD_ROOT}/../../src/yb/rpc/inbound_call.cc:236:13 (libyrpc.so+0xe7289)
    #18 yb::rpc::(anonymous namespace)::Worker::Execute() ${BUILD_ROOT}/../../src/yb/rpc/thread_pool.cc:104:15 (libyrpc.so+0x191524)
    #19 decltype(*std::declval<yb::rpc::(anonymous namespace)::Worker*&>().*std::declval<void (yb::rpc::(anonymous namespace)::Worker::*&)()>()()) std::__invoke[abi:v15003]<void (yb::rpc::(anonymous namespace)::Worker::*&)(), yb::rpc::(anonymous namespace)::Worker*&, void>(void (yb::rpc::(anonymous namespace)::Worker::*&)(), yb::rpc::(anonymous namespace)::Worker*&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/invoke.h:359:23 (libyrpc.so+0x19198b)
    #20 std::__bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), std::tuple<yb::rpc::(anonymous namespace)::Worker*>, std::tuple<>, __is_valid_bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), std::tuple<yb::rpc::(anonymous namespace)::Worker*>, std::tuple<>>::value>::type std::__apply_functor[abi:v15003]<void (yb::rpc::(anonymous namespace)::Worker::*)(), std::tuple<yb::rpc::(anonymous namespace)::Worker*>, 0ul, std::tuple<>>(void (yb::rpc::(anonymous namespace)::Worker::*&)(), std::tuple<yb::rpc::(anonymous namespace)::Worker*>&, std::__tuple_indices<0ul>, std::tuple<>&&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/bind.h:263:12 (libyrpc.so+0x19198b)
    #21 std::__bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), std::tuple<yb::rpc::(anonymous namespace)::Worker*>, std::tuple<>, __is_valid_bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), std::tuple<yb::rpc::(anonymous namespace)::Worker*>, std::tuple<>>::value>::type std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>::operator()[abi:v15003]<>() /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/bind.h:295:20 (libyrpc.so+0x19198b)
    #22 decltype(std::declval<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>()()) std::__invoke[abi:v15003]<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/invoke.h:394:23 (libyrpc.so+0x19198b)
    #23 void std::__invoke_void_return_wrapper<void, true>::__call<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/invoke.h:479:9 (libyrpc.so+0x19198b)
    #24 std::__function::__alloc_func<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>, std::allocator<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>>, void ()>::operator()[abi:v15003]() /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:185:16 (libyrpc.so+0x19198b)
    #25 std::__function::__func<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>, std::allocator<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>>, void ()>::operator()() /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:359:12 (libyrpc.so+0x19198b)
    #26 std::__function::__value_func<void ()>::operator()[abi:v15003]() const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:512:16 (libyb_util.so+0x354e74)
    #27 std::function<void ()>::operator()() const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__functional/function.h:1197:12 (libyb_util.so+0x354e74)
    #28 yb::Thread::SuperviseThread(void*) ${BUILD_ROOT}/../../src/yb/util/thread.cc:800:3 (libyb_util.so+0x354e74)

  Previous write of size 8 at 0x7b4800107e10 by thread T220 (mutexes: write M0):
    #0 operator delete(void*) /opt/yb-build/llvm/yb-llvm-v15.0.3-yb-1-1667341687-0b8d1183-centos7-x86_64-build/src/llvm-project/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126:3 (ql-tablet-test+0x12bece)
    #1 std::default_delete<yb::docdb::DocReadContext>::operator()[abi:v15003](yb::docdb::DocReadContext*) const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__memory/unique_ptr.h:48:5 (libtablet.so+0x2c570f)
    #2 std::unique_ptr<yb::docdb::DocReadContext, std::default_delete<yb::docdb::DocReadContext>>::reset[abi:v15003](yb::docdb::DocReadContext*) /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__memory/unique_ptr.h:305:7 (libtablet.so+0x2c570f)
    #3 std::unique_ptr<yb::docdb::DocReadContext, std::default_delete<yb::docdb::DocReadContext>>::~unique_ptr[abi:v15003]() /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20221115221900-32d6b89b02-centos7-x86_64-clang15/installed/tsan/libcxx/include/c++/v1/__memory/unique_ptr.h:259:19 (libtablet.so+0x2c570f)

Full logs: https://gist.github.com/def-/ef47ca773fa6d0e7f90fc2cc3d6f99a8