Adds pread_disk_io.cpp and pread_storage.cpp to CMakeList sources

CMakeLists.txt

@@ -166,6 +166,7 @@ set(libtorrent_aux_include_files
 	disable_warnings_pop.hpp
 	disable_warnings_push.hpp
 	disk_buffer_pool.hpp
+	disk_cache.hpp
 	disk_completed_queue.hpp
 	mmap_disk_job.hpp
 	disk_job.hpp
@@ -383,6 +384,8 @@ set(sources
 	posix_disk_io.cpp
 	posix_part_file.cpp
 	posix_storage.cpp
+	pread_disk_io.cpp
+	pread_storage.cpp
 	proxy_base.cpp
 	proxy_settings.cpp
 	puff.cpp

ChangeLog

@@ -1,3 +1,4 @@
+	* add a multi-threaded, pread()-based, disk I/O backend (pread_disk_io)
 	* made disk_interface's status_t type a flags type
 	* optimize resume data format to use less space
 	* add a simpler overload to bencode() returning a vector<char>

Jamfile

@@ -599,8 +599,8 @@ feature simulator : off on : composite propagated link-incompatible ;
 feature.compose <simulator>on : <define>TORRENT_BUILD_SIMULATOR ;
 
 feature invariant-checks : off on full : composite propagated link-incompatible ;
-feature.compose <invariant-checks>on : <define>TORRENT_USE_INVARIANT_CHECKS=1 ;
-feature.compose <invariant-checks>full : <define>TORRENT_USE_INVARIANT_CHECKS=1 <define>TORRENT_EXPENSIVE_INVARIANT_CHECKS ;
+feature.compose <invariant-checks>on : <define>TORRENT_USE_INVARIANT_CHECKS=1 <define>BOOST_MULTI_INDEX_ENABLE_INVARIANT_CHECKING=1 ;
+feature.compose <invariant-checks>full : <define>TORRENT_USE_INVARIANT_CHECKS=1 <define>TORRENT_EXPENSIVE_INVARIANT_CHECKS <define>BOOST_MULTI_INDEX_ENABLE_INVARIANT_CHECKING=1 ;
 
 feature utp-log : off on : composite propagated link-incompatible ;
 feature.compose <utp-log>on : <define>TORRENT_UTP_LOG_ENABLE ;
@@ -674,7 +674,7 @@ feature boost-link : default static shared : propagated composite ;
 # constructors on some containers when enabling debug iterators, so it's
 # possible to turn them off
 feature debug-iterators : default off on : composite propagated link-incompatible ;
-feature.compose <debug-iterators>on : <define>_GLIBCXX_DEBUG <define>_GLIBCXX_DEBUG_PEDANTIC ;
+feature.compose <debug-iterators>on : <define>_GLIBCXX_DEBUG <define>_GLIBCXX_DEBUG_PEDANTIC <defin>BOOST_MULTI_INDEX_ENABLE_SAFE_MODE=1 ;
 feature.compose <debug-iterators>off : <define>_ITERATOR_DEBUG_LEVEL=0 ;
 
 feature fpic : off on : composite propagated link-incompatible ;
@@ -876,6 +876,8 @@ SOURCES =
 	mmap
 	mmap_disk_io
 	mmap_storage
+	pread_disk_io
+	pread_storage
 	posix_disk_io
 	posix_part_file
 	posix_storage

Makefile

@@ -379,6 +379,8 @@ SOURCES = \
   posix_disk_io.cpp               \
   posix_part_file.cpp             \
   posix_storage.cpp               \
+  pread_disk_io.cpp               \
+  pread_storage.cpp               \
   proxy_base.cpp                  \
   proxy_settings.cpp              \
   puff.cpp                        \
@@ -560,6 +562,7 @@ HEADERS = \
   aux_/disable_warnings_pop.hpp     \
   aux_/disable_warnings_push.hpp    \
   aux_/disk_buffer_pool.hpp         \
+  aux_/disk_cache.hpp               \
   aux_/disk_completed_queue.hpp     \
   aux_/disk_io_thread_pool.hpp      \
   aux_/disk_job_fence.hpp           \
@@ -626,6 +629,7 @@ HEADERS = \
   aux_/portmap.hpp                  \
   aux_/posix_part_file.hpp          \
   aux_/posix_storage.hpp            \
+  aux_/pread_disk_job.hpp           \
   aux_/proxy_base.hpp               \
   aux_/proxy_settings.hpp           \
   aux_/puff.hpp                     \

bindings/python/src/session.cpp

@@ -28,6 +28,10 @@
 #include <libtorrent/extensions/ut_metadata.hpp>
 #include <libtorrent/extensions/ut_pex.hpp>
 
+#include <libtorrent/mmap_disk_io.hpp>
+#include <libtorrent/posix_disk_io.hpp>
+#include <libtorrent/pread_disk_io.hpp>
+
 namespace boost
 {
 	// this fixes mysterious link error on msvc
@@ -864,6 +868,27 @@ namespace
     }
 #endif
 
+    std::string get_disk_io(session_params const& s)
+    {
+        // this field is write-only
+        return "default_disk_io_constructor";
+    }
+
+    void set_disk_io(session_params& s, std::string disk_io)
+    {
+#if TORRENT_HAVE_MMAP || TORRENT_HAVE_MAP_VIEW_OF_FILE
+        if (disk_io == "mmap_disk_io_constructor")
+            s.disk_io_constructor = &lt::mmap_disk_io_constructor;
+        else
+#endif
+        if (disk_io == "posix_disk_io_constructor")
+            s.disk_io_constructor = &lt::posix_disk_io_constructor;
+        else if (disk_io == "pread_disk_io_constructor")
+            s.disk_io_constructor = &lt::pread_disk_io_constructor;
+        else
+            s.disk_io_constructor = &lt::default_disk_io_constructor;
+    }
+
 } // anonymous namespace
 
 struct dummy1 {};
@@ -1040,6 +1065,7 @@ void bind_session()
         .def_readwrite("dht_state", &session_params::dht_state)
 #endif
         .def_readwrite("ip_filter", &session_params::ip_filter)
+        .add_property("disk_io_constructor", &get_disk_io, &set_disk_io)
         ;
 
     def("read_session_params", &read_session_params_entry, (arg("dict"), arg("flags")=save_state_flags_t::all()));

docs/upgrade_to_2.1.rst

@@ -160,3 +160,10 @@ to allow reporting the oversized_file condition (which was introduced in
 The flags can be found in the ``lt::disk_status`` namespace.
 
 This change affects custom storage implementations.
+
+storage_params
+--------------
+
+The storage_params type has two new fields indicating whether the torrent has v1
+and/or v2 hashes. this allows disk I/O subsystems to know in advance whether
+block-hashes (v2 torrents) or flat piece hashes (v1 torrents) will be required.

examples/client_test.cpp

@@ -52,9 +52,13 @@ see LICENSE file.
 #include "libtorrent/write_resume_data.hpp"
 #include "libtorrent/string_view.hpp"
 #include "libtorrent/disk_interface.hpp" // for open_file_state
-#include "libtorrent/disabled_disk_io.hpp" // for disabled_disk_io_constructor
 #include "libtorrent/load_torrent.hpp"
 
+#include "libtorrent/mmap_disk_io.hpp"
+#include "libtorrent/posix_disk_io.hpp"
+#include "libtorrent/pread_disk_io.hpp"
+#include "libtorrent/disabled_disk_io.hpp"
+
 #include "torrent_view.hpp"
 #include "session_view.hpp"
 #include "print.hpp"
@@ -1342,7 +1346,10 @@ CLIENT OPTIONS
   -e <loops>            exit client after the specified number of iterations
                         through the main loop
   -O                    print session stats counters to the log
-  -1                    exit on first torrent completing (useful for benchmarks))"
+  -1                    exit on first torrent completing (useful for benchmarks)
+  -i <disk-io>          specify which disk I/O back-end to use. One of:
+                        mmap, posix, pread, disabled
+)"
 #ifdef TORRENT_UTP_LOG_ENABLE
 R"(
   -q                    Enable uTP transport-level verbose logging
@@ -1365,12 +1372,7 @@ BITTORRENT OPTIONS
 NETWORK OPTIONS
   -x <file>             loads an emule IP-filter file
   -Y                    Rate limit local peers
-)"
-#if TORRENT_USE_I2P
-R"(  -i <i2p-host>         the hostname to an I2P SAM bridge to use
-)"
-#endif
-R"(
+
 DISK OPTIONS
   -a <mode>             sets the allocation mode. [sparse|allocate]
   -0                    disable disk I/O, read garbage and don't flush to disk
@@ -1552,6 +1554,27 @@ int main(int argc, char* argv[])
 				break;
 			case 'T': max_connections_per_torrent = atoi(arg); break;
 			case 'r': peer = arg; break;
+			case 'i': {
+#if TORRENT_HAVE_MMAP || TORRENT_HAVE_MAP_VIEW_OF_FILE
+				if (arg == "mmap"_sv)
+					params.disk_io_constructor = lt::mmap_disk_io_constructor;
+				else
+#endif
+				if (arg == "posix"_sv)
+					params.disk_io_constructor = lt::posix_disk_io_constructor;
+#if TORRENT_HAVE_PREAD || defined TORRENT_WINDOWS
+				else if (arg == "pread"_sv)
+					params.disk_io_constructor = lt::pread_disk_io_constructor;
+#endif
+				else if (arg == "disabled"_sv)
+					params.disk_io_constructor = lt::disabled_disk_io_constructor;
+				else
+				{
+					std::fprintf(stderr, "unknown disk-io subsystem: \"%s\"\n", arg);
+					std::exit(1);
+				}
+				break;
+			}
 			case 'e':
 				{
 					loop_limit = atoi(arg);

examples/connection_tester.cpp

@@ -939,7 +939,9 @@ void generate_data(std::string const path, torrent_info const& ti)
 		path,
 		storage_mode_sparse,
 		priorities,
-		info_hash
+		info_hash,
+		ti.v1(),
+		ti.v2(),
 	};
 
 	storage_holder st = disk->new_torrent(params, std::shared_ptr<void>());

include/libtorrent/aux_/debug_disk_thread.hpp

@@ -24,6 +24,7 @@ see LICENSE file.
 #include <string>
 #include <sstream>
 #include <unordered_map>
+#include <thread>
 
 #include "libtorrent/aux_/disk_job.hpp"
 #include "libtorrent/disk_interface.hpp"
@@ -93,6 +94,10 @@ inline std::string print_job(aux::disk_job const& j)
 				<< " buf-offset:" << j.buffer_offset << " size:" << j.buffer_size << " )";
 		}
 
+		void operator()(job::kick_hasher const& j) const {
+			m_ss << "kick-hasher( piece:" << j.piece << " )";
+		}
+
 	private:
 		std::stringstream& m_ss;
 	};

include/libtorrent/aux_/disk_buffer_pool.hpp

@@ -21,6 +21,7 @@ see LICENSE file.
 #include <mutex>
 #include <functional>
 #include <memory>
+#include <optional>
 
 #include "libtorrent/io_context.hpp"
 #include "libtorrent/span.hpp"
@@ -54,6 +55,8 @@ namespace aux {
 			return m_in_use;
 		}
 
+		std::optional<int> flush_request() const;
+
 		void set_settings(settings_interface const& sett);
 
 	private: