Download raw body.
Patch: sysutils/py-ansible-libssh, stability backports from GitHub
Hi all.
I was experimenting with py-ansible-libssh, but I didn't went to far as
currently the port is affected by multiple issues:
- Fix pushing large files over SCP
https://github.com/ansible/pylibssh/pull/661
- Prevent SEGFAULTs on consecutive exec_command() invocations
https://github.com/ansible/pylibssh/pull/658
- Fix large SFTP reads
https://github.com/ansible/pylibssh/pull/638
- sftp.get downloads only the last chunk of file
https://github.com/ansible/pylibssh/issues/341
- SIGSEGV when running ssh_channel.exec_command() over different channels
https://github.com/ansible/pylibssh/issues/657
Finally all related pull request are merged to upstream's develop
branch. I am backporting here commits which I was affected by and
I cannot reproduce any issues I've seen before. Well, except
significantly slower SFTP trasnfers vs SCP, but at least they
work now.
I would like to see this in, as currently for me the port as is,
in the repo is not really usable.
Index: Makefile
===================================================================
RCS file: /cvs/ports/sysutils/py-ansible-libssh/Makefile,v
diff -u -p -u -r1.3 Makefile
--- Makefile 26 Dec 2024 16:15:11 -0000 1.3
+++ Makefile 27 Mar 2025 20:52:34 -0000
@@ -3,7 +3,7 @@ COMMENT = Python bindings for libssh sp
DISTNAME = ansible-pylibssh-${MODPY_DISTV}
PKGNAME = py-ansible-libssh-${MODPY_DISTV}
CATEGORIES = sysutils
-REVISION = 0
+REVISION = 1
MAINTAINER = Denis Fondras <denis@openbsd.org>
@@ -16,6 +16,8 @@ MODULES = lang/python
MODPY_PI = Yes
# uses own in-tree backend which wraps setuptools
MODPY_PYBUILD = setuptools_scm
+
+DEBUG_PACKAGES = ${BUILD_PACKAGES}
CFLAGS += -I${LOCALBASE}/include
BUILD_DEPENDS = sysutils/py-expandvars \
Index: patches/patch-src_pylibsshext_channel_pxd
===================================================================
RCS file: patches/patch-src_pylibsshext_channel_pxd
diff -N patches/patch-src_pylibsshext_channel_pxd
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_channel_pxd 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,13 @@
+git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/
+
+Index: src/pylibsshext/channel.pxd
+--- src/pylibsshext/channel.pxd.orig
++++ src/pylibsshext/channel.pxd
+@@ -23,3 +23,7 @@ cdef class Channel:
+ cdef _session
+ cdef libssh.ssh_channel _libssh_channel
+ cdef libssh.ssh_session _libssh_session
++
++cdef class ChannelCallback:
++ cdef callbacks.ssh_channel_callbacks_struct callback
++ cdef _userdata
Index: patches/patch-src_pylibsshext_channel_pyx
===================================================================
RCS file: patches/patch-src_pylibsshext_channel_pyx
diff -N patches/patch-src_pylibsshext_channel_pyx
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_channel_pyx 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,53 @@
+git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/
+
+Index: src/pylibsshext/channel.pyx
+--- src/pylibsshext/channel.pyx.orig
++++ src/pylibsshext/channel.pyx
+@@ -45,6 +45,16 @@ cdef int _process_outputs(libssh.ssh_session session,
+ result.stdout += data_b
+ return len
+
++cdef class ChannelCallback:
++ def __cinit__(self):
++ memset(&self.callback, 0, sizeof(self.callback))
++ callbacks.ssh_callbacks_init(&self.callback)
++ self.callback.channel_data_function = <callbacks.ssh_channel_data_callback>&_process_outputs
++
++ def set_user_data(self, userdata):
++ self._userdata = userdata
++ self.callback.userdata = <void *>self._userdata
++
+ cdef class Channel:
+ def __cinit__(self, session):
+ self._session = session
+@@ -159,19 +169,23 @@ cdef class Channel:
+ libssh.ssh_channel_free(channel)
+ raise LibsshChannelException("Failed to open_session: [{0}]".format(rc))
+
++ result = CompletedProcess(args=command, returncode=-1, stdout=b'', stderr=b'')
++
++ cb = ChannelCallback()
++ cb.set_user_data(result)
++ callbacks.ssh_set_channel_callbacks(channel, &cb.callback)
++ # keep the callback around in the session object to avoid use after free
++ self._session.push_callback(cb)
++
+ rc = libssh.ssh_channel_request_exec(channel, command.encode("utf-8"))
+ if rc != libssh.SSH_OK:
+ libssh.ssh_channel_close(channel)
+ libssh.ssh_channel_free(channel)
+ raise LibsshChannelException("Failed to execute command [{0}]: [{1}]".format(command, rc))
+- result = CompletedProcess(args=command, returncode=-1, stdout=b'', stderr=b'')
+
+- cdef callbacks.ssh_channel_callbacks_struct cb
+- memset(&cb, 0, sizeof(cb))
+- cb.channel_data_function = <callbacks.ssh_channel_data_callback>&_process_outputs
+- cb.userdata = <void *>result
+- callbacks.ssh_callbacks_init(&cb)
+- callbacks.ssh_set_channel_callbacks(channel, &cb)
++ # wait before remote writes all data before closing the channel
++ while not libssh.ssh_channel_is_eof(channel):
++ libssh.ssh_channel_poll(channel, 0)
+
+ libssh.ssh_channel_send_eof(channel)
+ result.returncode = libssh.ssh_channel_get_exit_status(channel)
Index: patches/patch-src_pylibsshext_includes_libssh_pxd
===================================================================
RCS file: patches/patch-src_pylibsshext_includes_libssh_pxd
diff -N patches/patch-src_pylibsshext_includes_libssh_pxd
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_includes_libssh_pxd 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,17 @@
+$ git -P log --oneline e9935cbcf62f7d2f83b9234f6db4e41a3ca480cb^..HEAD -- src/pylibsshext/includes/libssh.pxd
+265cae1 sftp: Increase SFTP chunk size to specification-recommended 32kB
+
+Index: src/pylibsshext/includes/libssh.pxd
+--- src/pylibsshext/includes/libssh.pxd.orig
++++ src/pylibsshext/includes/libssh.pxd
+@@ -27,6 +27,10 @@ cdef extern from "libssh/libssh.h" nogil:
+ pass
+ ctypedef ssh_session_struct* ssh_session
+
++ cdef struct ssh_string_struct:
++ pass
++ ctypedef ssh_string_struct* ssh_string
++
+ cdef struct ssh_key_struct:
+ pass
+ ctypedef ssh_key_struct* ssh_key
Index: patches/patch-src_pylibsshext_includes_sftp_pxd
===================================================================
RCS file: patches/patch-src_pylibsshext_includes_sftp_pxd
diff -N patches/patch-src_pylibsshext_includes_sftp_pxd
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_includes_sftp_pxd 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,61 @@
+$ git -P log --oneline e9935cbcf62f7d2f83b9234f6db4e41a3ca480cb^..HEAD -- src/pylibsshext/includes/sftp.pxd
+265cae1 sftp: Increase SFTP chunk size to specification-recommended 32kB
+
+Index: src/pylibsshext/includes/sftp.pxd
+--- src/pylibsshext/includes/sftp.pxd.orig
++++ src/pylibsshext/includes/sftp.pxd
+@@ -17,9 +17,11 @@
+ #
+ from posix.types cimport mode_t
+
+-from pylibsshext.includes.libssh cimport ssh_channel, ssh_session
++from libc cimport stdint
+
++from pylibsshext.includes.libssh cimport ssh_channel, ssh_session, ssh_string
+
++
+ cdef extern from "libssh/sftp.h" nogil:
+
+ struct sftp_session_struct:
+@@ -30,6 +32,31 @@ cdef extern from "libssh/sftp.h" nogil:
+ pass
+ ctypedef sftp_file_struct * sftp_file
+
++ struct sftp_attributes_struct:
++ char *name
++ char *longname
++ stdint.uint32_t flags
++ stdint.uint8_t type
++ stdint.uint64_t size
++ stdint.uint32_t uid
++ stdint.uint32_t gid
++ char *owner
++ char *group
++ stdint.uint32_t permissions
++ stdint.uint64_t atime64
++ stdint.uint32_t atime
++ stdint.uint32_t atime_nseconds
++ stdint.uint64_t createtime
++ stdint.uint32_t createtime_nseconds
++ stdint.uint64_t mtime64
++ stdint.uint32_t mtime
++ stdint.uint32_t mtime_nseconds
++ ssh_string acl
++ stdint.uint32_t extended_count
++ ssh_string extended_type
++ ssh_string extended_data
++ ctypedef sftp_attributes_struct * sftp_attributes
++
+ cdef int SSH_FX_OK
+ cdef int SSH_FX_EOF
+ cdef int SSH_FX_NO_SUCH_FILE
+@@ -54,6 +81,9 @@ cdef extern from "libssh/sftp.h" nogil:
+ ssize_t sftp_write(sftp_file file, const void *buf, size_t count)
+ ssize_t sftp_read(sftp_file file, const void *buf, size_t count)
+ int sftp_get_error(sftp_session sftp)
++
++ sftp_attributes sftp_stat(sftp_session session, const char *path)
++
+
+ cdef extern from "sys/stat.h" nogil:
+ cdef int S_IRWXU
Index: patches/patch-src_pylibsshext_scp_pyx
===================================================================
RCS file: patches/patch-src_pylibsshext_scp_pyx
diff -N patches/patch-src_pylibsshext_scp_pyx
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_scp_pyx 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,48 @@
+$ git -P log --oneline a5292bdce6948e5eda91d6ff7008c7cc0f912054^..HEAD -- src/pylibsshext/scp.pyx
+63ab9a4 scp: Make the chunk size constant more readable
+a5292bd scp: Fix pushing large files
+
+https://github.com/ansible/pylibssh/pull/661
+
+Index: src/pylibsshext/scp.pyx
+--- src/pylibsshext/scp.pyx.orig
++++ src/pylibsshext/scp.pyx
+@@ -24,7 +24,7 @@ from pylibsshext.errors cimport LibsshSCPException
+ from pylibsshext.session cimport get_libssh_session
+
+
+-SCP_MAX_CHUNK = 65536
++SCP_MAX_CHUNK = 65_536 # 64kB
+
+
+ cdef class SCP:
+@@ -74,15 +74,25 @@ cdef class SCP:
+ )
+
+ try:
++ # Read buffer
++ read_buffer_size = min(file_size, SCP_MAX_CHUNK)
++
+ # Begin to send to the file
+ rc = libssh.ssh_scp_push_file(scp, filename_b, file_size, file_mode)
+ if rc != libssh.SSH_OK:
+ raise LibsshSCPException("Can't open remote file: %s" % self._get_ssh_error_str())
+
+- # Write to the open file
+- rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(f.read()), file_size)
+- if rc != libssh.SSH_OK:
+- raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str())
++ remaining_bytes_to_read = file_size
++ while remaining_bytes_to_read > 0:
++ # Read the chunk from local file
++ read_bytes = min(remaining_bytes_to_read, read_buffer_size)
++ read_buffer = f.read(read_bytes)
++
++ # Write to the open file
++ rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(read_buffer), read_bytes)
++ if rc != libssh.SSH_OK:
++ raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str())
++ remaining_bytes_to_read -= read_bytes
+ finally:
+ libssh.ssh_scp_close(scp)
+ libssh.ssh_scp_free(scp)
Index: patches/patch-src_pylibsshext_session_pxd
===================================================================
RCS file: patches/patch-src_pylibsshext_session_pxd
diff -N patches/patch-src_pylibsshext_session_pxd
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_session_pxd 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,12 @@
+git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/
+
+Index: src/pylibsshext/session.pxd
+--- src/pylibsshext/session.pxd.orig
++++ src/pylibsshext/session.pxd
+@@ -26,5 +26,6 @@ cdef class Session:
+ cdef _hash_py
+ cdef _fingerprint_py
+ cdef _keytype_py
++ cdef _channel_callbacks
+
+ cdef libssh.ssh_session get_libssh_session(Session session)
Index: patches/patch-src_pylibsshext_session_pyx
===================================================================
RCS file: patches/patch-src_pylibsshext_session_pyx
diff -N patches/patch-src_pylibsshext_session_pyx
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_session_pyx 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,26 @@
+git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/
+
+Index: src/pylibsshext/session.pyx
+--- src/pylibsshext/session.pyx.orig
++++ src/pylibsshext/session.pyx
+@@ -107,6 +107,10 @@ cdef class Session(object):
+ self._hash_py = None
+ self._fingerprint_py = None
+ self._keytype_py = None
++ # Due to delayed freeing of channels, some older libssh versions might expect
++ # the callbacks to be around even after we free the underlying channels so
++ # we should free them only when we terminate the session.
++ self._channel_callbacks = []
+
+ def __cinit__(self, host=None, **kwargs):
+ self._libssh_session = libssh.ssh_new()
+@@ -122,6 +126,9 @@ cdef class Session(object):
+ libssh.ssh_disconnect(self._libssh_session)
+ libssh.ssh_free(self._libssh_session)
+ self._libssh_session = NULL
++
++ def push_callback(self, callback):
++ self._channel_callbacks.append(callback)
+
+ @property
+ def port(self):
Index: patches/patch-src_pylibsshext_sftp_pyx
===================================================================
RCS file: patches/patch-src_pylibsshext_sftp_pyx
diff -N patches/patch-src_pylibsshext_sftp_pyx
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_pylibsshext_sftp_pyx 27 Mar 2025 20:52:34 -0000
@@ -0,0 +1,106 @@
+$ git -P log --oneline e9935cbcf62f7d2f83b9234f6db4e41a3ca480cb^..HEAD -- src/pylibsshext/sftp.pyx
+265cae1 sftp: Increase SFTP chunk size to specification-recommended 32kB
+d3678e7 sftp: Fix overwriting existing files while downloading
+e9935cb sftp: Fix downloading files
+
+Index: src/pylibsshext/sftp.pyx
+--- src/pylibsshext/sftp.pyx.orig
++++ src/pylibsshext/sftp.pyx
+@@ -18,11 +18,15 @@
+ from posix.fcntl cimport O_CREAT, O_RDONLY, O_TRUNC, O_WRONLY
+
+ from cpython.bytes cimport PyBytes_AS_STRING
++from cpython.mem cimport PyMem_Free, PyMem_Malloc
+
+ from pylibsshext.errors cimport LibsshSFTPException
+ from pylibsshext.session cimport get_libssh_session
+
+
++SFTP_MAX_CHUNK = 32_768 # 32kB
++
++
+ MSG_MAP = {
+ sftp.SSH_FX_OK: "No error",
+ sftp.SSH_FX_EOF: "End-of-file encountered",
+@@ -63,7 +67,7 @@ cdef class SFTP:
+ rf = sftp.sftp_open(self._libssh_sftp_session, remote_file_b, O_WRONLY | O_CREAT | O_TRUNC, sftp.S_IRWXU)
+ if rf is NULL:
+ raise LibsshSFTPException("Opening remote file [%s] for write failed with error [%s]" % (remote_file, self._get_sftp_error_str()))
+- buffer = f.read(1024)
++ buffer = f.read(SFTP_MAX_CHUNK)
+
+ while buffer != b"":
+ length = len(buffer)
+@@ -76,38 +80,54 @@ cdef class SFTP:
+ self._get_sftp_error_str(),
+ )
+ )
+- buffer = f.read(1024)
++ buffer = f.read(SFTP_MAX_CHUNK)
+ sftp.sftp_close(rf)
+
+ def get(self, remote_file, local_file):
+ cdef sftp.sftp_file rf
+- cdef char read_buffer[1024]
++ cdef char *read_buffer = NULL
++ cdef sftp.sftp_attributes attrs
+
+ remote_file_b = remote_file
+ if isinstance(remote_file_b, unicode):
+ remote_file_b = remote_file.encode("utf-8")
+
++ attrs = sftp.sftp_stat(self._libssh_sftp_session, remote_file_b)
++ if attrs is NULL:
++ raise LibsshSFTPException("Failed to stat the remote file [%s]. Error: [%s]"
++ % (remote_file, self._get_sftp_error_str()))
++ file_size = attrs.size
++
+ rf = sftp.sftp_open(self._libssh_sftp_session, remote_file_b, O_RDONLY, sftp.S_IRWXU)
+ if rf is NULL:
+ raise LibsshSFTPException("Opening remote file [%s] for read failed with error [%s]" % (remote_file, self._get_sftp_error_str()))
+
+- while True:
+- file_data = sftp.sftp_read(rf, <void *>read_buffer, sizeof(char) * 1024)
+- if file_data == 0:
+- break
+- elif file_data < 0:
+- sftp.sftp_close(rf)
+- raise LibsshSFTPException("Reading data from remote file [%s] failed with error [%s]"
+- % (remote_file, self._get_sftp_error_str()))
++ try:
++ with open(local_file, 'wb') as f:
++ buffer_size = min(SFTP_MAX_CHUNK, file_size)
++ read_buffer = <char *>PyMem_Malloc(buffer_size)
++ if read_buffer is NULL:
++ raise LibsshSFTPException("Memory allocation error")
+
+- with open(local_file, 'wb+') as f:
+- bytes_written = f.write(read_buffer[:file_data])
+- if bytes_written and file_data != bytes_written:
+- sftp.sftp_close(rf)
+- raise LibsshSFTPException("Number of bytes [%s] read from remote file [%s]"
+- " does not match number of bytes [%s] written to local file [%s]"
+- " due to error [%s]"
+- % (file_data, remote_file, bytes_written, local_file, self._get_sftp_error_str()))
++ while True:
++ file_data = sftp.sftp_read(rf, <void *>read_buffer, sizeof(char) * buffer_size)
++ if file_data == 0:
++ break
++ elif file_data < 0:
++ sftp.sftp_close(rf)
++ raise LibsshSFTPException("Reading data from remote file [%s] failed with error [%s]"
++ % (remote_file, self._get_sftp_error_str()))
++
++ bytes_written = f.write(read_buffer[:file_data])
++ if bytes_written and file_data != bytes_written:
++ sftp.sftp_close(rf)
++ raise LibsshSFTPException("Number of bytes [%s] read from remote file [%s]"
++ " does not match number of bytes [%s] written to local file [%s]"
++ " due to error [%s]"
++ % (file_data, remote_file, bytes_written, local_file, self._get_sftp_error_str()))
++ finally:
++ if read_buffer is not NULL:
++ PyMem_Free(read_buffer)
+ sftp.sftp_close(rf)
+
+ def close(self):
--
Regards,
Mikolaj
Patch: sysutils/py-ansible-libssh, stability backports from GitHub