From de18d8cd62d9440b1c66a660c760b0f549b5b830 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Wed, 18 Dec 2024 01:12:49 +0100 Subject: [PATCH 1/9] Add `+LINUX` and `+WINDOWS` doctest options This allows to selectively run tests only on a single platform. We can add `# doctest: +LINUX` comments to tests that cannot work on Windows and the other way around. To easily skip a lot of tests the `doctest_additional_flags` global variable can be defined in a `testsetup`. This is achieved by monkey patching sphinx doctest's DocTestBuilder to use our own DocTestRunner which removes examples from the tests that have flags that don't match the platform we're running on. --- docs/source/conf.py | 62 ++++++++++++++++++++++++++++++++---- docs/source/elf/corefile.rst | 3 ++ docs/source/gdb.rst | 3 ++ pwnlib/context/__init__.py | 9 ++++-- 4 files changed, 67 insertions(+), 10 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index d908e2436..2466613f6 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -398,6 +398,11 @@ def dont_skip_any_doctests(app, what, name, obj, skip, options): class _DummyClass(object): pass +# doctest optionflags for platform-specific tests +# they are skipped on other platforms +WINDOWS = doctest.register_optionflag('WINDOWS') +LINUX = doctest.register_optionflag('LINUX') + class Py2OutputChecker(_DummyClass, doctest.OutputChecker): def check_output(self, want, got, optionflags): sup = super(Py2OutputChecker, self).check_output @@ -425,6 +430,41 @@ def check_output(self, want, got, optionflags): return False return True +import sphinx.ext.doctest + +class PlatformDocTestRunner(sphinx.ext.doctest.SphinxDocTestRunner): + def run(self, test, compileflags=None, out=None, clear_globs=True): + original_optionflags = self.optionflags | test.globs.get('doctest_additional_flags', 0) + def filter_platform(example): + optionflags = original_optionflags + if example.options: + for (optionflag, val) in example.options.items(): + if val: + optionflags |= optionflag + else: + optionflags &= ~optionflag + + if (optionflags & WINDOWS) == WINDOWS and sys.platform != 'win32': + return False + if (optionflags & LINUX) == LINUX and sys.platform != 'linux': + return False + return True + + test.examples[:] = [example for example in test.examples if filter_platform(example)] + + return super(PlatformDocTestRunner, self).run(test, compileflags, out, clear_globs) + +class PlatformDocTestBuilder(sphinx.ext.doctest.DocTestBuilder): + _test_runner = None + + @property + def test_runner(self): + return self._test_runner + + @test_runner.setter + def test_runner(self, value): + self._test_runner = PlatformDocTestRunner(value._checker, value._verbose, value.optionflags) + def py2_doctest_init(self, checker=None, verbose=None, optionflags=0): if checker is None: checker = Py2OutputChecker() @@ -432,10 +472,10 @@ def py2_doctest_init(self, checker=None, verbose=None, optionflags=0): if 'doctest' in sys.argv: def setup(app): - pass # app.connect('autodoc-skip-member', dont_skip_any_doctests) + app.add_builder(PlatformDocTestBuilder, override=True) + # app.connect('autodoc-skip-member', dont_skip_any_doctests) if sys.version_info[:1] < (3,): - import sphinx.ext.doctest sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init else: # monkey patching paramiko due to https://github.com/paramiko/paramiko/pull/1661 @@ -444,8 +484,16 @@ def setup(app): paramiko.client.hexlify = lambda x: binascii.hexlify(x).decode() paramiko.util.safe_string = lambda x: '' # function result never *actually used* class EndlessLoop(Exception): pass - def alrm_handler(sig, frame): - signal.alarm(180) # three minutes - raise EndlessLoop() - signal.signal(signal.SIGALRM, alrm_handler) - signal.alarm(600) # ten minutes + if sys.platform == 'win32': + def alrm_handler(): + raise EndlessLoop() + import threading + timer = threading.Timer(600, alrm_handler) + timer.daemon = True + timer.start() + else: + def alrm_handler(sig, frame): + signal.alarm(180) # three minutes + raise EndlessLoop() + signal.signal(signal.SIGALRM, alrm_handler) + signal.alarm(600) # ten minutes diff --git a/docs/source/elf/corefile.rst b/docs/source/elf/corefile.rst index ab088414e..bdbe046a1 100644 --- a/docs/source/elf/corefile.rst +++ b/docs/source/elf/corefile.rst @@ -18,6 +18,9 @@ # Set the environment here so it's not in the middle of our tests. os.environ.setdefault('SHELL', '/bin/sh') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.elf.corefile` --- Core Files =========================================================== diff --git a/docs/source/gdb.rst b/docs/source/gdb.rst index a2067e956..99b2263aa 100644 --- a/docs/source/gdb.rst +++ b/docs/source/gdb.rst @@ -4,6 +4,9 @@ context.arch = 'amd64' context.terminal = [os.path.join(os.path.dirname(pwnlib.__file__), 'gdb_faketerminal.py')] + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.gdb` --- Working with GDB ====================================== diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index a95ec4310..4efb6b8aa 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -324,7 +324,7 @@ class ContextType(object): >>> _=(thread.start(), thread.join()) 90 >>> # Pwnthread uses the correct context from creation-time - >>> _=(pwnthread.start(), pwnthread.join()) + >>> _=(pwnthread.start(), pwnthread.join()) # doctest: +LINUX 00000000 >>> nop() 00f020e3 @@ -870,6 +870,9 @@ def binary(self, binary): Examples: + .. doctest:: + :options: +LINUX + >>> context.clear() >>> context.arch, context.bits ('i386', 32) @@ -1078,7 +1081,7 @@ def log_console(self, stream): >>> context.log_level = 'warn' >>> log.warn("Hello") [!] Hello - >>> context.log_console=open('/dev/null', 'w') + >>> context.log_console=open(os.devnull, 'w') >>> log.warn("Hello") >>> context.clear() """ @@ -1405,7 +1408,7 @@ def cache_dir(self): True >>> os.chmod(cache_dir, 0o000) >>> context.cache_dir = True - >>> context.cache_dir is None + >>> context.cache_dir is None # doctest: +LINUX True >>> os.chmod(cache_dir, 0o755) >>> cache_dir == context.cache_dir From 062b5b9efab97cd97e20bf945a37f982db14c5e2 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Wed, 18 Dec 2024 02:16:20 +0100 Subject: [PATCH 2/9] Limit Sphinx version to secure platform patches Avoid major versions which might change the API. We have to check if the platform optionflags still work on newer versions once they are available. --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f9da2e525..c63977f47 100755 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -19,6 +19,6 @@ psutil requests>=2.5.1 ropgadget>=5.3 sphinx==1.8.6; python_version<'3' -sphinx>=7.0.0; python_version>='3' +sphinx>=8.1.3, <9; python_version>='3' sphinx_rtd_theme sphinxcontrib-autoprogram<=0.1.5 From 4c657b169b0b42613db30e1d44bf80392d66034b Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Wed, 18 Dec 2024 02:00:23 +0100 Subject: [PATCH 3/9] CI: Run doctests with coverage on Windows Disable all non-trivial tests on Windows for now. The goal is to reduce the amount of linux-only tests. --- .github/workflows/ci.yml | 17 ++++++++++++++++- docs/source/adb.rst | 3 +++ docs/source/asm.rst | 3 +++ docs/source/elf/elf.rst | 3 +++ docs/source/encoders.rst | 3 +++ docs/source/filesystem.rst | 3 +++ docs/source/intro.rst | 3 +++ docs/source/libcdb.rst | 3 +++ docs/source/qemu.rst | 3 +++ docs/source/rop/rop.rst | 3 +++ docs/source/rop/srop.rst | 3 +++ docs/source/runner.rst | 3 +++ docs/source/shellcraft.rst | 3 +++ docs/source/shellcraft/aarch64.rst | 3 +++ docs/source/shellcraft/amd64.rst | 3 +++ docs/source/shellcraft/arm.rst | 3 +++ docs/source/shellcraft/i386.rst | 3 +++ docs/source/shellcraft/mips.rst | 3 +++ docs/source/shellcraft/riscv64.rst | 3 +++ docs/source/shellcraft/thumb.rst | 3 +++ docs/source/tubes.rst | 3 +++ docs/source/tubes/buffer.rst | 3 +++ docs/source/tubes/processes.rst | 3 +++ docs/source/tubes/serial.rst | 3 +++ docs/source/tubes/sockets.rst | 3 +++ docs/source/tubes/ssh.rst | 3 +++ docs/source/ui.rst | 3 +++ docs/source/util/net.rst | 3 +++ docs/source/util/proc.rst | 3 +++ pwnlib/context/__init__.py | 6 +++++- pwnlib/fmtstr.py | 3 +++ pwnlib/memleak.py | 3 +++ pwnlib/rop/ret2dlresolve.py | 12 ++++++------ pwnlib/util/iters.py | 3 +++ pwnlib/util/misc.py | 6 ++++-- pwnlib/util/packing.py | 4 ++++ pwnlib/util/sh_string.py | 23 +++++++++++++---------- 37 files changed, 141 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca1da827c..14c49b36a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,14 +284,29 @@ jobs: pip install --upgrade pip pip install --upgrade --editable . + - name: Install documentation dependencies + run: pip install -r docs/requirements.txt + - name: Sanity checks run: | python -bb -c 'from pwn import *' python -bb examples/text.py + + - name: Coverage doctests + run: | + python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest + + - uses: actions/upload-artifact@v4 + with: + name: coverage-windows + path: .coverage* + include-hidden-files: true upload-coverage: runs-on: ubuntu-latest - needs: test + needs: + - test + - windows-test steps: - uses: actions/checkout@v4 with: diff --git a/docs/source/adb.rst b/docs/source/adb.rst index 242979fab..baf1f492c 100644 --- a/docs/source/adb.rst +++ b/docs/source/adb.rst @@ -4,6 +4,9 @@ from pwn import * adb = pwnlib.adb + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.adb` --- Android Debug Bridge ===================================================== diff --git a/docs/source/asm.rst b/docs/source/asm.rst index a47bd867c..2dccfcc13 100644 --- a/docs/source/asm.rst +++ b/docs/source/asm.rst @@ -4,6 +4,9 @@ import subprocess from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.asm` --- Assembler functions ========================================= diff --git a/docs/source/elf/elf.rst b/docs/source/elf/elf.rst index b54e9a393..be7e915fd 100644 --- a/docs/source/elf/elf.rst +++ b/docs/source/elf/elf.rst @@ -5,6 +5,9 @@ from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT import shutil + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.elf.elf` --- ELF Files =========================================================== diff --git a/docs/source/encoders.rst b/docs/source/encoders.rst index e36ed86d4..ad9806a15 100644 --- a/docs/source/encoders.rst +++ b/docs/source/encoders.rst @@ -1,6 +1,9 @@ .. testsetup:: * from pwn import * + + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] :mod:`pwnlib.encoders` --- Encoding Shellcode =============================================== diff --git a/docs/source/filesystem.rst b/docs/source/filesystem.rst index 6a7fae504..d80ddb40c 100644 --- a/docs/source/filesystem.rst +++ b/docs/source/filesystem.rst @@ -6,6 +6,9 @@ from pwnlib.tubes.ssh import ssh from pwnlib.filesystem import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.filesystem` --- Manipulating Files Locally and Over SSH ==================================================================== diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 25e5cc1ae..7ead5ad88 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -2,6 +2,9 @@ from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + Getting Started ======================== diff --git a/docs/source/libcdb.rst b/docs/source/libcdb.rst index 54d152a58..8536fada5 100644 --- a/docs/source/libcdb.rst +++ b/docs/source/libcdb.rst @@ -3,6 +3,9 @@ from pwn import * from pwnlib.libcdb import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.libcdb` --- Libc Database =========================================== diff --git a/docs/source/qemu.rst b/docs/source/qemu.rst index a28ab316c..b8f28b23f 100644 --- a/docs/source/qemu.rst +++ b/docs/source/qemu.rst @@ -2,6 +2,9 @@ from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.qemu` --- QEMU Utilities ========================================== diff --git a/docs/source/rop/rop.rst b/docs/source/rop/rop.rst index 8d5d93f39..944467630 100644 --- a/docs/source/rop/rop.rst +++ b/docs/source/rop/rop.rst @@ -19,6 +19,9 @@ context.clear() + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.rop.rop` --- Return Oriented Programming ========================================================== diff --git a/docs/source/rop/srop.rst b/docs/source/rop/srop.rst index f38490498..8248be2f5 100644 --- a/docs/source/rop/srop.rst +++ b/docs/source/rop/srop.rst @@ -7,6 +7,9 @@ from pwnlib.elf import ELF from pwnlib.tubes.process import process + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.rop.srop` --- Sigreturn Oriented Programming ========================================================== diff --git a/docs/source/runner.rst b/docs/source/runner.rst index 2aa661aeb..008db6bab 100644 --- a/docs/source/runner.rst +++ b/docs/source/runner.rst @@ -3,6 +3,9 @@ from pwnlib.runner import * from pwnlib.asm import asm + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.runner` --- Running Shellcode =========================================== diff --git a/docs/source/shellcraft.rst b/docs/source/shellcraft.rst index 1f5d0bc7a..c00031bf3 100644 --- a/docs/source/shellcraft.rst +++ b/docs/source/shellcraft.rst @@ -2,6 +2,9 @@ from pwnlib import shellcraft + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft` --- Shellcode generation ================================================= diff --git a/docs/source/shellcraft/aarch64.rst b/docs/source/shellcraft/aarch64.rst index 1abf6f68c..700d30f2e 100644 --- a/docs/source/shellcraft/aarch64.rst +++ b/docs/source/shellcraft/aarch64.rst @@ -3,6 +3,9 @@ from pwn import * context.clear(arch='aarch64') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.aarch64` --- Shellcode for AArch64 =========================================================== diff --git a/docs/source/shellcraft/amd64.rst b/docs/source/shellcraft/amd64.rst index 8aced2c41..c63cccc0c 100644 --- a/docs/source/shellcraft/amd64.rst +++ b/docs/source/shellcraft/amd64.rst @@ -3,6 +3,9 @@ from pwn import * context.clear(arch='amd64') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.amd64` --- Shellcode for AMD64 =========================================================== diff --git a/docs/source/shellcraft/arm.rst b/docs/source/shellcraft/arm.rst index 8e4d2400e..96316900f 100644 --- a/docs/source/shellcraft/arm.rst +++ b/docs/source/shellcraft/arm.rst @@ -3,6 +3,9 @@ from pwn import * context.clear(arch='arm') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.arm` --- Shellcode for ARM =========================================================== diff --git a/docs/source/shellcraft/i386.rst b/docs/source/shellcraft/i386.rst index 5820abf18..6cf72dcd4 100644 --- a/docs/source/shellcraft/i386.rst +++ b/docs/source/shellcraft/i386.rst @@ -3,6 +3,9 @@ from pwn import * context.clear(arch='i386') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.i386` --- Shellcode for Intel 80386 =========================================================== diff --git a/docs/source/shellcraft/mips.rst b/docs/source/shellcraft/mips.rst index 15e9f3e0d..5efb5c736 100644 --- a/docs/source/shellcraft/mips.rst +++ b/docs/source/shellcraft/mips.rst @@ -12,6 +12,9 @@ context.clear(arch='mips') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.mips` --- Shellcode for MIPS =========================================================== diff --git a/docs/source/shellcraft/riscv64.rst b/docs/source/shellcraft/riscv64.rst index 6e4a01148..4b0c3cf11 100644 --- a/docs/source/shellcraft/riscv64.rst +++ b/docs/source/shellcraft/riscv64.rst @@ -3,6 +3,9 @@ from pwn import * context.clear(arch='riscv64') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.riscv64` --- Shellcode for RISCV64 ========================================================== diff --git a/docs/source/shellcraft/thumb.rst b/docs/source/shellcraft/thumb.rst index 8beaddbe6..020f368fa 100644 --- a/docs/source/shellcraft/thumb.rst +++ b/docs/source/shellcraft/thumb.rst @@ -3,6 +3,9 @@ from pwn import * context.clear(arch='thumb') + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.shellcraft.thumb` --- Shellcode for Thumb Mode =========================================================== diff --git a/docs/source/tubes.rst b/docs/source/tubes.rst index af8d197ac..291a18768 100644 --- a/docs/source/tubes.rst +++ b/docs/source/tubes.rst @@ -2,6 +2,9 @@ from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.tubes` --- Talking to the World! ============================================= diff --git a/docs/source/tubes/buffer.rst b/docs/source/tubes/buffer.rst index a90be2f51..db115d546 100644 --- a/docs/source/tubes/buffer.rst +++ b/docs/source/tubes/buffer.rst @@ -2,6 +2,9 @@ from pwnlib.tubes.buffer import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.tubes.buffer` --- buffer implementation for tubes ============================================================== diff --git a/docs/source/tubes/processes.rst b/docs/source/tubes/processes.rst index a14377c0b..b46e0bef3 100644 --- a/docs/source/tubes/processes.rst +++ b/docs/source/tubes/processes.rst @@ -2,6 +2,9 @@ from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.tubes.process` --- Processes =========================================================== diff --git a/docs/source/tubes/serial.rst b/docs/source/tubes/serial.rst index d850a0483..cc52f9d71 100644 --- a/docs/source/tubes/serial.rst +++ b/docs/source/tubes/serial.rst @@ -2,6 +2,9 @@ from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.tubes.serialtube` --- Serial Ports =========================================================== diff --git a/docs/source/tubes/sockets.rst b/docs/source/tubes/sockets.rst index bf5967b47..616f344fc 100644 --- a/docs/source/tubes/sockets.rst +++ b/docs/source/tubes/sockets.rst @@ -3,6 +3,9 @@ from pwn import * from pwnlib.tubes.server import server + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.tubes.sock` --- Sockets =========================================================== diff --git a/docs/source/tubes/ssh.rst b/docs/source/tubes/ssh.rst index ae351cc3b..2de8bad79 100644 --- a/docs/source/tubes/ssh.rst +++ b/docs/source/tubes/ssh.rst @@ -2,6 +2,9 @@ from pwn import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.tubes.ssh` --- SSH =========================================================== diff --git a/docs/source/ui.rst b/docs/source/ui.rst index c0fb38394..62be83e00 100644 --- a/docs/source/ui.rst +++ b/docs/source/ui.rst @@ -3,6 +3,9 @@ from pwn import * import io + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.ui` --- Functions for user interaction =================================================== diff --git a/docs/source/util/net.rst b/docs/source/util/net.rst index 1c1ca8fbb..cf4247020 100644 --- a/docs/source/util/net.rst +++ b/docs/source/util/net.rst @@ -2,6 +2,9 @@ from pwnlib.util.net import * + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.util.net` --- Networking interfaces =================================================== diff --git a/docs/source/util/proc.rst b/docs/source/util/proc.rst index b556f25aa..521143222 100644 --- a/docs/source/util/proc.rst +++ b/docs/source/util/proc.rst @@ -4,6 +4,9 @@ from pwnlib.tubes.process import process import os, sys + import doctest + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + :mod:`pwnlib.util.proc` --- Working with ``/proc/`` =================================================== diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index 4efb6b8aa..e4c24ea45 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -308,6 +308,10 @@ class ContextType(object): 'little' >>> context.bits 32 + + .. doctest:: + :options: +LINUX + >>> def nop(): ... print(enhex(pwnlib.asm.asm('nop'))) >>> nop() @@ -324,7 +328,7 @@ class ContextType(object): >>> _=(thread.start(), thread.join()) 90 >>> # Pwnthread uses the correct context from creation-time - >>> _=(pwnthread.start(), pwnthread.join()) # doctest: +LINUX + >>> _=(pwnthread.start(), pwnthread.join()) 00000000 >>> nop() 00f020e3 diff --git a/pwnlib/fmtstr.py b/pwnlib/fmtstr.py index 97a534916..88fb39e2e 100644 --- a/pwnlib/fmtstr.py +++ b/pwnlib/fmtstr.py @@ -33,6 +33,9 @@ We can automate the exploitation of the process like so: +.. doctest:: + :options: +LINUX + >>> program = pwnlib.data.elf.fmtstr.get('i386') >>> def exec_fmt(payload): ... p = process(program) diff --git a/pwnlib/memleak.py b/pwnlib/memleak.py index 4759f3486..35525eafa 100644 --- a/pwnlib/memleak.py +++ b/pwnlib/memleak.py @@ -39,6 +39,9 @@ def some_leaker(addr): Example: + .. doctest:: + :options: +LINUX + >>> import pwnlib >>> binsh = pwnlib.util.misc.read('/bin/sh') >>> @pwnlib.memleak.MemLeak diff --git a/pwnlib/rop/ret2dlresolve.py b/pwnlib/rop/ret2dlresolve.py index 08a05420a..c05fa9e67 100644 --- a/pwnlib/rop/ret2dlresolve.py +++ b/pwnlib/rop/ret2dlresolve.py @@ -32,9 +32,9 @@ 0x0014: 0x2b84 [dlresolve index] 0x0018: b'gaaa' 0x001c: 0x804ae24 arg0 - >>> p = elf.process() - >>> p.sendline(fit({64+context.bytes*3: raw_rop, 200: dlresolve.payload})) - >>> p.recvline() + >>> p = elf.process() # doctest: +LINUX + >>> p.sendline(fit({64+context.bytes*3: raw_rop, 200: dlresolve.payload})) # doctest: +LINUX + >>> p.recvline() # doctest: +LINUX b'pwned\n' You can also use ``Ret2dlresolve`` on AMD64: @@ -56,9 +56,9 @@ 0x0038: 0x601e48 [arg0] rdi = 6299208 0x0040: 0x4003e0 [plt_init] system 0x0048: 0x15670 [dlresolve index] - >>> p = elf.process() - >>> p.sendline(fit({64+context.bytes: raw_rop, 200: dlresolve.payload})) - >>> if dlresolve.unreliable: + >>> p = elf.process() # doctest: +LINUX + >>> p.sendline(fit({64+context.bytes: raw_rop, 200: dlresolve.payload})) # doctest: +LINUX + >>> if dlresolve.unreliable: # doctest: +LINUX ... p.poll(True) == -signal.SIGSEGV ... else: ... p.recvline() == b'pwned\n' diff --git a/pwnlib/util/iters.py b/pwnlib/util/iters.py index d037fe921..726eb7913 100644 --- a/pwnlib/util/iters.py +++ b/pwnlib/util/iters.py @@ -888,6 +888,9 @@ def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = Example: + .. doctest:: + :options: +LINUX + >>> mbruteforce(lambda x: x == 'hello', string.ascii_lowercase, length = 10) 'hello' >>> mbruteforce(lambda x: x == 'hello', 'hlo', 5, 'downfrom') is None diff --git a/pwnlib/util/misc.py b/pwnlib/util/misc.py index e55465d33..d0fe8639f 100644 --- a/pwnlib/util/misc.py +++ b/pwnlib/util/misc.py @@ -123,7 +123,7 @@ def read(path, count=-1, skip=0): Examples: - >>> read('/proc/self/exe')[:4] + >>> read('/proc/self/exe')[:4] # doctest: +LINUX b'\x7fELF' """ path = os.path.expanduser(os.path.expandvars(path)) @@ -163,8 +163,10 @@ def which(name, all = False, path=None): Example: - >>> which('sh') # doctest: +ELLIPSIS + >>> which('sh') # doctest: +ELLIPSIS +LINUX '.../bin/sh' + >>> which('cmd') # doctest: +ELLIPSIS +WINDOWS + '...\\cmd.EXE' """ # If name is a path, do not attempt to resolve it. if os.path.sep in name: diff --git a/pwnlib/util/packing.py b/pwnlib/util/packing.py index 3503bd937..3fca19457 100644 --- a/pwnlib/util/packing.py +++ b/pwnlib/util/packing.py @@ -988,6 +988,10 @@ def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False): ('H', 'e', 'l', 'l', 'o', b'?') >>> dd(list('Hello!'), (63,), skip = 5) ['H', 'e', 'l', 'l', 'o', b'?'] + + .. doctest:: + :options: +LINUX + >>> _ = open('/tmp/foo', 'w').write('A' * 10) >>> dd(open('/tmp/foo'), open('/dev/zero'), skip = 3, count = 4).read() 'AAA\\x00\\x00\\x00\\x00AAA' diff --git a/pwnlib/util/sh_string.py b/pwnlib/util/sh_string.py index 00ddb81fc..b10931bd2 100644 --- a/pwnlib/util/sh_string.py +++ b/pwnlib/util/sh_string.py @@ -280,16 +280,19 @@ def test_all(): def test(original): r"""Tests the output provided by a shell interpreting a string - >>> test(b'foobar') - >>> test(b'foo bar') - >>> test(b'foo bar\n') - >>> test(b"foo'bar") - >>> test(b"foo\\\\bar") - >>> test(b"foo\\\\'bar") - >>> test(b"foo\\x01'bar") - >>> test(b'\n') - >>> test(b'\xff') - >>> test(os.urandom(16 * 1024).replace(b'\x00', b'')) + .. doctest:: + :options: +LINUX + + >>> test(b'foobar') + >>> test(b'foo bar') + >>> test(b'foo bar\n') + >>> test(b"foo'bar") + >>> test(b"foo\\\\bar") + >>> test(b"foo\\\\'bar") + >>> test(b"foo\\x01'bar") + >>> test(b'\n') + >>> test(b'\xff') + >>> test(os.urandom(16 * 1024).replace(b'\x00', b'')) """ input = sh_string(original) From 3f22d819385c48f3a7a0f978113ca4400f87a006 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Wed, 18 Dec 2024 02:07:34 +0100 Subject: [PATCH 4/9] Only apply platform patch on Python 3 --- docs/source/conf.py | 6 +++--- pwnlib/util/misc.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2466613f6..4c42e8cef 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -471,13 +471,13 @@ def py2_doctest_init(self, checker=None, verbose=None, optionflags=0): doctest.DocTestRunner.__init__(self, checker, verbose, optionflags) if 'doctest' in sys.argv: - def setup(app): - app.add_builder(PlatformDocTestBuilder, override=True) - # app.connect('autodoc-skip-member', dont_skip_any_doctests) if sys.version_info[:1] < (3,): sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init else: + def setup(app): + app.add_builder(PlatformDocTestBuilder, override=True) + # app.connect('autodoc-skip-member', dont_skip_any_doctests) # monkey patching paramiko due to https://github.com/paramiko/paramiko/pull/1661 import paramiko.client import binascii diff --git a/pwnlib/util/misc.py b/pwnlib/util/misc.py index d0fe8639f..75790c596 100644 --- a/pwnlib/util/misc.py +++ b/pwnlib/util/misc.py @@ -165,8 +165,6 @@ def which(name, all = False, path=None): >>> which('sh') # doctest: +ELLIPSIS +LINUX '.../bin/sh' - >>> which('cmd') # doctest: +ELLIPSIS +WINDOWS - '...\\cmd.EXE' """ # If name is a path, do not attempt to resolve it. if os.path.sep in name: From 5ad47238b91ef0fcc8493a0580aac6c0e58bff42 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Wed, 18 Dec 2024 02:31:31 +0100 Subject: [PATCH 5/9] Disable uploading coverage on Windows The handrolled coveralls upload cannot handle mixed operating systems. Refs #2480 --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14c49b36a..e1211f80d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,17 +296,17 @@ jobs: run: | python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest - - uses: actions/upload-artifact@v4 - with: - name: coverage-windows - path: .coverage* - include-hidden-files: true + # FIXME: Paths are broken when uploading coverage on ubuntu + # coverage.exceptions.NoSource: No source for code: '/home/runner/work/pwntools/pwntools/D:\a\pwntools\pwntools\pwn\__init__.py'. + # - uses: actions/upload-artifact@v4 + # with: + # name: coverage-windows + # path: .coverage* + # include-hidden-files: true upload-coverage: runs-on: ubuntu-latest - needs: - - test - - windows-test + needs: test steps: - uses: actions/checkout@v4 with: From d80b0968b0e35c2dfa6767850683dd2b866f14c8 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Wed, 18 Dec 2024 11:51:09 +0100 Subject: [PATCH 6/9] Use threading.Timer for doctest timeout To interrupt the code running on the main thread, we send a signal using `_thread.interrupt_main()`. By default this causes a KeyboardInterrupt exception, which might be handled explicitly. To raise an explicit EndlessLoop exception inside the code that is taking too long, register a SIGABRT signal handler which raises the EndlessLoop exception. The exception from the signal handler is added to the call stack and handled by the code currently running. This allows to print a better stack trace on timeout. It is the same concept as the old implementation using `signal.alarm` but platform agnostic. https://anonbadger.wordpress.com/2018/12/15/python-signal-handlers-and-exceptions/ --- docs/source/conf.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 4c42e8cef..d658bc574 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -483,17 +483,24 @@ def setup(app): import binascii paramiko.client.hexlify = lambda x: binascii.hexlify(x).decode() paramiko.util.safe_string = lambda x: '' # function result never *actually used* - class EndlessLoop(Exception): pass - if sys.platform == 'win32': - def alrm_handler(): - raise EndlessLoop() - import threading - timer = threading.Timer(600, alrm_handler) + class EndlessLoop(BaseException): pass + def sigabrt_handler(signum, frame): + raise EndlessLoop() + # thread.interrupt_main received the signum parameter in Python 3.10 + if sys.version_info >= (3, 10): + signal.signal(signal.SIGABRT, sigabrt_handler) + def alrm_handler(): + try: + import thread + except ImportError: + import _thread as thread + # pre Python 3.10 this raises a KeyboardInterrupt in the main thread. + # it might not show a traceback in that case, but it will stop the endless loop. + thread.interrupt_main(signal.SIGABRT) + timer = threading.Timer(interval=180, function=alrm_handler) # three minutes timer.daemon = True timer.start() - else: - def alrm_handler(sig, frame): - signal.alarm(180) # three minutes - raise EndlessLoop() - signal.signal(signal.SIGALRM, alrm_handler) - signal.alarm(600) # ten minutes + import threading + timer = threading.Timer(interval=600, function=alrm_handler) # ten minutes + timer.daemon = True + timer.start() \ No newline at end of file From c92fe20a304570e8bf4d8082a7154da8a0777c74 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Sun, 22 Dec 2024 12:01:25 +0100 Subject: [PATCH 7/9] Add POSIX optionflag Run test on other UNIX systems too if they don't use Linux specifics. Add a TODO optionflag too to mark platform restrictions that might be too strict and should be looked at. --- docs/source/asm.rst | 3 ++- docs/source/conf.py | 6 ++++++ docs/source/elf/corefile.rst | 2 +- docs/source/elf/elf.rst | 3 ++- docs/source/encoders.rst | 3 ++- docs/source/filesystem.rst | 3 ++- docs/source/gdb.rst | 3 ++- docs/source/intro.rst | 2 +- docs/source/libcdb.rst | 3 ++- docs/source/qemu.rst | 3 ++- docs/source/rop/rop.rst | 1 + docs/source/runner.rst | 3 ++- docs/source/shellcraft.rst | 3 ++- docs/source/shellcraft/amd64.rst | 1 + docs/source/shellcraft/i386.rst | 2 +- docs/source/tubes.rst | 3 ++- docs/source/tubes/buffer.rst | 3 ++- docs/source/tubes/processes.rst | 3 ++- docs/source/tubes/serial.rst | 3 ++- docs/source/tubes/sockets.rst | 3 ++- docs/source/tubes/ssh.rst | 3 ++- docs/source/ui.rst | 2 +- pwnlib/context/__init__.py | 6 +++--- pwnlib/fmtstr.py | 2 +- pwnlib/memleak.py | 2 +- pwnlib/util/iters.py | 2 +- pwnlib/util/misc.py | 4 ++-- pwnlib/util/packing.py | 2 +- pwnlib/util/sh_string.py | 2 +- 29 files changed, 52 insertions(+), 29 deletions(-) diff --git a/docs/source/asm.rst b/docs/source/asm.rst index 2dccfcc13..d87b14333 100644 --- a/docs/source/asm.rst +++ b/docs/source/asm.rst @@ -4,8 +4,9 @@ import subprocess from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.asm` --- Assembler functions ========================================= diff --git a/docs/source/conf.py b/docs/source/conf.py index d658bc574..37cfbd93a 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -402,6 +402,10 @@ class _DummyClass(object): pass # they are skipped on other platforms WINDOWS = doctest.register_optionflag('WINDOWS') LINUX = doctest.register_optionflag('LINUX') +POSIX = doctest.register_optionflag('POSIX') + +# doctest optionflag for tests that haven't been looked at yet +TODO = doctest.register_optionflag('TODO') class Py2OutputChecker(_DummyClass, doctest.OutputChecker): def check_output(self, want, got, optionflags): @@ -448,6 +452,8 @@ def filter_platform(example): return False if (optionflags & LINUX) == LINUX and sys.platform != 'linux': return False + if (optionflags & POSIX) == POSIX and os.name != 'posix': + return False return True test.examples[:] = [example for example in test.examples if filter_platform(example)] diff --git a/docs/source/elf/corefile.rst b/docs/source/elf/corefile.rst index bdbe046a1..85668ae79 100644 --- a/docs/source/elf/corefile.rst +++ b/docs/source/elf/corefile.rst @@ -19,7 +19,7 @@ os.environ.setdefault('SHELL', '/bin/sh') import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.elf.corefile` --- Core Files diff --git a/docs/source/elf/elf.rst b/docs/source/elf/elf.rst index be7e915fd..7501ca20c 100644 --- a/docs/source/elf/elf.rst +++ b/docs/source/elf/elf.rst @@ -5,8 +5,9 @@ from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT import shutil + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.elf.elf` --- ELF Files =========================================================== diff --git a/docs/source/encoders.rst b/docs/source/encoders.rst index ad9806a15..8132023f4 100644 --- a/docs/source/encoders.rst +++ b/docs/source/encoders.rst @@ -2,8 +2,9 @@ from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.encoders` --- Encoding Shellcode =============================================== diff --git a/docs/source/filesystem.rst b/docs/source/filesystem.rst index d80ddb40c..26cc62ce3 100644 --- a/docs/source/filesystem.rst +++ b/docs/source/filesystem.rst @@ -6,8 +6,9 @@ from pwnlib.tubes.ssh import ssh from pwnlib.filesystem import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.filesystem` --- Manipulating Files Locally and Over SSH ==================================================================== diff --git a/docs/source/gdb.rst b/docs/source/gdb.rst index 99b2263aa..5f4f30406 100644 --- a/docs/source/gdb.rst +++ b/docs/source/gdb.rst @@ -4,8 +4,9 @@ context.arch = 'amd64' context.terminal = [os.path.join(os.path.dirname(pwnlib.__file__), 'gdb_faketerminal.py')] + # TODO: Test on cygwin too import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.gdb` --- Working with GDB ====================================== diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 7ead5ad88..3832dde32 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -3,7 +3,7 @@ from pwn import * import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] Getting Started ======================== diff --git a/docs/source/libcdb.rst b/docs/source/libcdb.rst index 8536fada5..a31dd7eb4 100644 --- a/docs/source/libcdb.rst +++ b/docs/source/libcdb.rst @@ -3,8 +3,9 @@ from pwn import * from pwnlib.libcdb import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.libcdb` --- Libc Database =========================================== diff --git a/docs/source/qemu.rst b/docs/source/qemu.rst index b8f28b23f..bf8884de2 100644 --- a/docs/source/qemu.rst +++ b/docs/source/qemu.rst @@ -2,8 +2,9 @@ from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.qemu` --- QEMU Utilities diff --git a/docs/source/rop/rop.rst b/docs/source/rop/rop.rst index 944467630..553cac97d 100644 --- a/docs/source/rop/rop.rst +++ b/docs/source/rop/rop.rst @@ -19,6 +19,7 @@ context.clear() + # TODO: Remove global LINUX flag import doctest doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] diff --git a/docs/source/runner.rst b/docs/source/runner.rst index 008db6bab..f0c33d7e8 100644 --- a/docs/source/runner.rst +++ b/docs/source/runner.rst @@ -3,8 +3,9 @@ from pwnlib.runner import * from pwnlib.asm import asm + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.runner` --- Running Shellcode =========================================== diff --git a/docs/source/shellcraft.rst b/docs/source/shellcraft.rst index c00031bf3..5d9e37c24 100644 --- a/docs/source/shellcraft.rst +++ b/docs/source/shellcraft.rst @@ -2,8 +2,9 @@ from pwnlib import shellcraft + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.shellcraft` --- Shellcode generation ================================================= diff --git a/docs/source/shellcraft/amd64.rst b/docs/source/shellcraft/amd64.rst index c63cccc0c..27c65547c 100644 --- a/docs/source/shellcraft/amd64.rst +++ b/docs/source/shellcraft/amd64.rst @@ -3,6 +3,7 @@ from pwn import * context.clear(arch='amd64') + # TODO: POSIX/WINDOWS shellcode test import doctest doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] diff --git a/docs/source/shellcraft/i386.rst b/docs/source/shellcraft/i386.rst index 6cf72dcd4..3d72adc5c 100644 --- a/docs/source/shellcraft/i386.rst +++ b/docs/source/shellcraft/i386.rst @@ -4,7 +4,7 @@ context.clear(arch='i386') import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.shellcraft.i386` --- Shellcode for Intel 80386 =========================================================== diff --git a/docs/source/tubes.rst b/docs/source/tubes.rst index 291a18768..02f46ec0f 100644 --- a/docs/source/tubes.rst +++ b/docs/source/tubes.rst @@ -2,8 +2,9 @@ from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.tubes` --- Talking to the World! ============================================= diff --git a/docs/source/tubes/buffer.rst b/docs/source/tubes/buffer.rst index db115d546..30adaf480 100644 --- a/docs/source/tubes/buffer.rst +++ b/docs/source/tubes/buffer.rst @@ -2,8 +2,9 @@ from pwnlib.tubes.buffer import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.tubes.buffer` --- buffer implementation for tubes ============================================================== diff --git a/docs/source/tubes/processes.rst b/docs/source/tubes/processes.rst index b46e0bef3..c4d891b35 100644 --- a/docs/source/tubes/processes.rst +++ b/docs/source/tubes/processes.rst @@ -2,8 +2,9 @@ from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.tubes.process` --- Processes =========================================================== diff --git a/docs/source/tubes/serial.rst b/docs/source/tubes/serial.rst index cc52f9d71..599ca35c4 100644 --- a/docs/source/tubes/serial.rst +++ b/docs/source/tubes/serial.rst @@ -2,8 +2,9 @@ from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.tubes.serialtube` --- Serial Ports =========================================================== diff --git a/docs/source/tubes/sockets.rst b/docs/source/tubes/sockets.rst index 616f344fc..9198794c0 100644 --- a/docs/source/tubes/sockets.rst +++ b/docs/source/tubes/sockets.rst @@ -3,8 +3,9 @@ from pwn import * from pwnlib.tubes.server import server + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.tubes.sock` --- Sockets =========================================================== diff --git a/docs/source/tubes/ssh.rst b/docs/source/tubes/ssh.rst index 2de8bad79..21c78e2f1 100644 --- a/docs/source/tubes/ssh.rst +++ b/docs/source/tubes/ssh.rst @@ -2,8 +2,9 @@ from pwn import * + # TODO: Remove global POSIX flag import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.tubes.ssh` --- SSH =========================================================== diff --git a/docs/source/ui.rst b/docs/source/ui.rst index 62be83e00..173f39bf9 100644 --- a/docs/source/ui.rst +++ b/docs/source/ui.rst @@ -4,7 +4,7 @@ import io import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['LINUX'] + doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] :mod:`pwnlib.ui` --- Functions for user interaction =================================================== diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index e4c24ea45..82f71c315 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -310,7 +310,7 @@ class ContextType(object): 32 .. doctest:: - :options: +LINUX + :options: +POSIX +TODO >>> def nop(): ... print(enhex(pwnlib.asm.asm('nop'))) @@ -875,7 +875,7 @@ def binary(self, binary): Examples: .. doctest:: - :options: +LINUX + :options: +POSIX +TODO >>> context.clear() >>> context.arch, context.bits @@ -1412,7 +1412,7 @@ def cache_dir(self): True >>> os.chmod(cache_dir, 0o000) >>> context.cache_dir = True - >>> context.cache_dir is None # doctest: +LINUX + >>> context.cache_dir is None # doctest: +POSIX +TODO True >>> os.chmod(cache_dir, 0o755) >>> cache_dir == context.cache_dir diff --git a/pwnlib/fmtstr.py b/pwnlib/fmtstr.py index 88fb39e2e..975efb250 100644 --- a/pwnlib/fmtstr.py +++ b/pwnlib/fmtstr.py @@ -34,7 +34,7 @@ We can automate the exploitation of the process like so: .. doctest:: - :options: +LINUX + :options: +POSIX +TODO >>> program = pwnlib.data.elf.fmtstr.get('i386') >>> def exec_fmt(payload): diff --git a/pwnlib/memleak.py b/pwnlib/memleak.py index 35525eafa..49909c19b 100644 --- a/pwnlib/memleak.py +++ b/pwnlib/memleak.py @@ -40,7 +40,7 @@ def some_leaker(addr): Example: .. doctest:: - :options: +LINUX + :options: +POSIX +TODO >>> import pwnlib >>> binsh = pwnlib.util.misc.read('/bin/sh') diff --git a/pwnlib/util/iters.py b/pwnlib/util/iters.py index 726eb7913..a044e3079 100644 --- a/pwnlib/util/iters.py +++ b/pwnlib/util/iters.py @@ -889,7 +889,7 @@ def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = Example: .. doctest:: - :options: +LINUX + :options: +POSIX +TODO >>> mbruteforce(lambda x: x == 'hello', string.ascii_lowercase, length = 10) 'hello' diff --git a/pwnlib/util/misc.py b/pwnlib/util/misc.py index 75790c596..f2e111edd 100644 --- a/pwnlib/util/misc.py +++ b/pwnlib/util/misc.py @@ -123,7 +123,7 @@ def read(path, count=-1, skip=0): Examples: - >>> read('/proc/self/exe')[:4] # doctest: +LINUX + >>> read('/proc/self/exe')[:4] # doctest: +LINUX +TODO b'\x7fELF' """ path = os.path.expanduser(os.path.expandvars(path)) @@ -163,7 +163,7 @@ def which(name, all = False, path=None): Example: - >>> which('sh') # doctest: +ELLIPSIS +LINUX + >>> which('sh') # doctest: +ELLIPSIS +POSIX +TODO '.../bin/sh' """ # If name is a path, do not attempt to resolve it. diff --git a/pwnlib/util/packing.py b/pwnlib/util/packing.py index 3fca19457..03d93bd37 100644 --- a/pwnlib/util/packing.py +++ b/pwnlib/util/packing.py @@ -990,7 +990,7 @@ def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False): ['H', 'e', 'l', 'l', 'o', b'?'] .. doctest:: - :options: +LINUX + :options: +POSIX +TODO >>> _ = open('/tmp/foo', 'w').write('A' * 10) >>> dd(open('/tmp/foo'), open('/dev/zero'), skip = 3, count = 4).read() diff --git a/pwnlib/util/sh_string.py b/pwnlib/util/sh_string.py index b10931bd2..52699edd2 100644 --- a/pwnlib/util/sh_string.py +++ b/pwnlib/util/sh_string.py @@ -281,7 +281,7 @@ def test(original): r"""Tests the output provided by a shell interpreting a string .. doctest:: - :options: +LINUX + :options: +POSIX >>> test(b'foobar') >>> test(b'foo bar') From fa1f3bd0311b76f47aa6b758333340f7ad8d222b Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Mon, 23 Dec 2024 19:29:26 +0100 Subject: [PATCH 8/9] Enable tube and tube/sockets tests on Windows --- docs/source/tubes.rst | 4 ---- docs/source/tubes/buffer.rst | 4 ---- docs/source/tubes/sockets.rst | 4 ---- pwnlib/tubes/listen.py | 17 +++++++++------ pwnlib/tubes/tube.py | 41 +++++++++++++++++++---------------- 5 files changed, 32 insertions(+), 38 deletions(-) diff --git a/docs/source/tubes.rst b/docs/source/tubes.rst index 02f46ec0f..af8d197ac 100644 --- a/docs/source/tubes.rst +++ b/docs/source/tubes.rst @@ -2,10 +2,6 @@ from pwn import * - # TODO: Remove global POSIX flag - import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] - :mod:`pwnlib.tubes` --- Talking to the World! ============================================= diff --git a/docs/source/tubes/buffer.rst b/docs/source/tubes/buffer.rst index 30adaf480..a90be2f51 100644 --- a/docs/source/tubes/buffer.rst +++ b/docs/source/tubes/buffer.rst @@ -2,10 +2,6 @@ from pwnlib.tubes.buffer import * - # TODO: Remove global POSIX flag - import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] - :mod:`pwnlib.tubes.buffer` --- buffer implementation for tubes ============================================================== diff --git a/docs/source/tubes/sockets.rst b/docs/source/tubes/sockets.rst index 9198794c0..bf5967b47 100644 --- a/docs/source/tubes/sockets.rst +++ b/docs/source/tubes/sockets.rst @@ -3,10 +3,6 @@ from pwn import * from pwnlib.tubes.server import server - # TODO: Remove global POSIX flag - import doctest - doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX'] - :mod:`pwnlib.tubes.sock` --- Sockets =========================================================== diff --git a/pwnlib/tubes/listen.py b/pwnlib/tubes/listen.py index ecbd629bd..08bae02c8 100644 --- a/pwnlib/tubes/listen.py +++ b/pwnlib/tubes/listen.py @@ -36,13 +36,16 @@ class listen(sock): >>> r.recvline() b'Hello\n' - >>> # It works with ipv4 by default - >>> l = listen() - >>> l.spawn_process('/bin/sh') - >>> r = remote('127.0.0.1', l.lport) - >>> r.sendline(b'echo Goodbye') - >>> r.recvline() - b'Goodbye\n' + .. doctest:: + :options: +POSIX +TODO + + >>> # It works with ipv4 by default + >>> l = listen() + >>> l.spawn_process('/bin/sh') + >>> r = remote('127.0.0.1', l.lport) + >>> r.sendline(b'echo Goodbye') + >>> r.recvline() + b'Goodbye\n' >>> # and it works with ipv6 by defaut, too! >>> l = listen() diff --git a/pwnlib/tubes/tube.py b/pwnlib/tubes/tube.py index 532045444..22fa29cb8 100644 --- a/pwnlib/tubes/tube.py +++ b/pwnlib/tubes/tube.py @@ -1166,26 +1166,29 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ Examples: - >>> l = listen() - >>> l.spawn_process('/bin/sh') - >>> r = remote('127.0.0.1', l.lport) - >>> r.upload_manually(b'some\\xca\\xfedata\\n', prompt=b'', chmod_flags='') - >>> r.sendline(b'cat ./payload') - >>> r.recvline() - b'some\\xca\\xfedata\\n' - - >>> r.upload_manually(cyclic(0x1000), target_path='./cyclic_pattern', prompt=b'', chunk_size=0x10, compression='gzip') - >>> r.sendline(b'sha256sum ./cyclic_pattern') - >>> r.recvlineS(keepends=False).startswith(sha256sumhex(cyclic(0x1000))) - True + .. doctest:: + :options: +POSIX +TODO + + >>> l = listen() + >>> l.spawn_process('/bin/sh') + >>> r = remote('127.0.0.1', l.lport) + >>> r.upload_manually(b'some\\xca\\xfedata\\n', prompt=b'', chmod_flags='') + >>> r.sendline(b'cat ./payload') + >>> r.recvline() + b'some\\xca\\xfedata\\n' + + >>> r.upload_manually(cyclic(0x1000), target_path='./cyclic_pattern', prompt=b'', chunk_size=0x10, compression='gzip') + >>> r.sendline(b'sha256sum ./cyclic_pattern') + >>> r.recvlineS(keepends=False).startswith(sha256sumhex(cyclic(0x1000))) + True - >>> blob = ELF.from_assembly(shellcraft.echo('Hello world!\\n') + shellcraft.exit(0)) - >>> r.upload_manually(blob.data, prompt=b'') - >>> r.sendline(b'./payload') - >>> r.recvline() - b'Hello world!\\n' - >>> r.close() - >>> l.close() + >>> blob = ELF.from_assembly(shellcraft.echo('Hello world!\\n') + shellcraft.exit(0)) + >>> r.upload_manually(blob.data, prompt=b'') + >>> r.sendline(b'./payload') + >>> r.recvline() + b'Hello world!\\n' + >>> r.close() + >>> l.close() """ echo_end = "" if not prompt: From cf4f7be5b9570e7fd5cb8bf2834e4f08ea9b6766 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Mon, 23 Dec 2024 21:05:49 +0100 Subject: [PATCH 9/9] Use `signal.alarm` for timeouts if it's available --- docs/source/conf.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 37cfbd93a..b77734c80 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -489,24 +489,31 @@ def setup(app): import binascii paramiko.client.hexlify = lambda x: binascii.hexlify(x).decode() paramiko.util.safe_string = lambda x: '' # function result never *actually used* - class EndlessLoop(BaseException): pass - def sigabrt_handler(signum, frame): - raise EndlessLoop() - # thread.interrupt_main received the signum parameter in Python 3.10 - if sys.version_info >= (3, 10): - signal.signal(signal.SIGABRT, sigabrt_handler) - def alrm_handler(): - try: - import thread - except ImportError: - import _thread as thread - # pre Python 3.10 this raises a KeyboardInterrupt in the main thread. - # it might not show a traceback in that case, but it will stop the endless loop. - thread.interrupt_main(signal.SIGABRT) - timer = threading.Timer(interval=180, function=alrm_handler) # three minutes + class EndlessLoop(Exception): pass + if hasattr(signal, 'alarm'): + def alrm_handler(sig, frame): + signal.alarm(180) # three minutes + raise EndlessLoop() + signal.signal(signal.SIGALRM, alrm_handler) + signal.alarm(600) # ten minutes + else: + def sigabrt_handler(signum, frame): + raise EndlessLoop() + # thread.interrupt_main received the signum parameter in Python 3.10 + if sys.version_info >= (3, 10): + signal.signal(signal.SIGABRT, sigabrt_handler) + def alrm_handler(): + try: + import thread + except ImportError: + import _thread as thread + # pre Python 3.10 this raises a KeyboardInterrupt in the main thread. + # it might not show a traceback in that case, but it will stop the endless loop. + thread.interrupt_main(signal.SIGABRT) + timer = threading.Timer(interval=180, function=alrm_handler) # three minutes + timer.daemon = True + timer.start() + import threading + timer = threading.Timer(interval=600, function=alrm_handler) # ten minutes timer.daemon = True timer.start() - import threading - timer = threading.Timer(interval=600, function=alrm_handler) # ten minutes - timer.daemon = True - timer.start() \ No newline at end of file