From 071e5958a5290b314c44d5d501744dd4156bb07d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 10:20:23 +0000 Subject: [PATCH 1/5] build(deps): bump setuptools-scm from 8.1.0 to 9.2.0 Bumps [setuptools-scm](https://github.com/pypa/setuptools-scm) from 8.1.0 to 9.2.0. - [Release notes](https://github.com/pypa/setuptools-scm/releases) - [Changelog](https://github.com/pypa/setuptools-scm/blob/main/CHANGELOG.md) - [Commits](https://github.com/pypa/setuptools-scm/compare/v8.1.0...v9.2.0) --- updated-dependencies: - dependency-name: setuptools-scm dependency-version: 9.2.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 73 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 38fae3af5..ecb4ea526 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -130,6 +130,14 @@ imagesize==1.4.1 \ --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a # via sphinx +importlib-metadata==8.7.0 \ + --hash=sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000 \ + --hash=sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd + # via sphinx +importlib-resources==6.5.2 \ + --hash=sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c \ + --hash=sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec + # via towncrier incremental==24.7.2 \ --hash=sha256:8cb2c3431530bec48ad70513931a760f446ad6c25e8333ca5d95e24b0ed7b8fe \ --hash=sha256:fb4f1d47ee60efe87d4f6f0ebb5f70b9760db2b2574c59c8e8912be4ebd464c9 @@ -220,7 +228,7 @@ mdurl==0.1.2 \ myst-parser==3.0.1 \ --hash=sha256:6457aaa33a5d474aca678b8ead9b3dc298e89c68e67012e73146ea6fd54babf1 \ --hash=sha256:88f0cb406cb363b077d176b51c476f62d60604d68a8dcdf4832e080441301a87 - # via -r requirements.in + # via -r docs/requirements.in packaging==24.1 \ --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 @@ -296,10 +304,10 @@ requests==2.32.4 \ --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 # via sphinx -setuptools-scm==8.3.1 \ - --hash=sha256:332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3 \ - --hash=sha256:3d555e92b75dacd037d32bafdf94f97af51ea29ae8c7b234cf94b7a5bd242a63 - # via -r requirements.in +setuptools-scm==9.2.0 \ + --hash=sha256:6662c9b9497b6c9bf13bead9d7a9084756f68238302c5ed089fb4dbd29d102d7 \ + --hash=sha256:c551ef54e2270727ee17067881c9687ca2aedf179fa5b8f3fab9e8d73bdc421f + # via -r docs/requirements.in snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a @@ -308,7 +316,7 @@ sphinx==6.2.1 \ --hash=sha256:6d56a34697bb749ffa0152feafc4b19836c755d90a7c59b72bc7dfd371b9cc6b \ --hash=sha256:97787ff1fa3256a3eef9eda523a63dbf299f7b47e053cfcf684a1c2a8380c912 # via - # -r requirements.in + # -r docs/requirements.in # myst-parser # sphinx-ansible-theme # sphinx-rtd-theme @@ -317,7 +325,7 @@ sphinx==6.2.1 \ sphinx-ansible-theme==0.9.1 \ --hash=sha256:242cdad5c5bcc12e910b2ad9dc7c12f8c5d96e640f8b5676a011148997a70395 \ --hash=sha256:4a41345123ed739976cd310c8afa8c7799e311c2e4fc1ecd2739293ad5da89eb - # via -r requirements.in + # via -r docs/requirements.in sphinx-rtd-theme==0.5.1 \ --hash=sha256:eda689eda0c7301a80cf122dad28b1861e5605cbf455558f3775e1e8200e83a5 \ --hash=sha256:fa6bebd5ab9a73da8e102509a86f3fcc36dec04a0b52ea80e5a033b2aba00113 @@ -325,7 +333,7 @@ sphinx-rtd-theme==0.5.1 \ sphinxcontrib-apidoc==0.6.0 \ --hash=sha256:329b9810d66988f48e127a6bd18cc8efbbd1cd20b8deb4691a35738af49ad88d \ --hash=sha256:668592f933eee858f3bc0d0810d56d50dfa0a70f650a2faaaad501b9a3504633 - # via -r requirements.in + # via -r docs/requirements.in sphinxcontrib-applehelp==2.0.0 \ --hash=sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1 \ --hash=sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5 @@ -353,15 +361,62 @@ sphinxcontrib-serializinghtml==2.0.0 \ sphinxcontrib-towncrier==0.5.0a0 \ --hash=sha256:11d130c3ad5e4649821d543c4ea7ab64bbe78df4d859ef94f4298e7845dc0f59 \ --hash=sha256:294e69df6e275e7a86df7ea6a927cc7c28c2c370a884cd5c45de6ec989858f27 - # via -r requirements.in + # via -r docs/requirements.in +tomli==2.2.1 \ + --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ + --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \ + --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \ + --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \ + --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \ + --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \ + --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \ + --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \ + --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \ + --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \ + --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \ + --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \ + --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \ + --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \ + --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \ + --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \ + --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \ + --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \ + --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \ + --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \ + --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \ + --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \ + --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \ + --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \ + --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \ + --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \ + --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \ + --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \ + --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \ + --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \ + --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ + --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 + # via + # incremental + # setuptools-scm + # towncrier towncrier==23.11.0 \ --hash=sha256:13937c247e3f8ae20ac44d895cf5f96a60ad46cfdcc1671759530d7837d9ee5d \ --hash=sha256:2e519ca619426d189e3c98c99558fe8be50c9ced13ea1fc20a4a353a95d2ded7 # via sphinxcontrib-towncrier +typing-extensions==4.14.1 \ + --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ + --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 + # via setuptools-scm urllib3==2.5.0 \ --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via requests +zipp==3.23.0 \ + --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ + --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: setuptools==80.9.0 \ From 5ecb5288a72ae4739c1092f7dab29df8d1f8f9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=87=BA=F0=9F=87=A6=20Sviatoslav=20Sydorenko=20=28?= =?UTF-8?q?=D0=A1=D0=B2=D1=8F=D1=82=D0=BE=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1?= =?UTF-8?q?=D0=B8=D0=B4=D0=BE=D1=80=D0=B5=D0=BD=D0=BA=D0=BE=29?= Date: Mon, 18 Aug 2025 14:26:40 +0200 Subject: [PATCH 2/5] Keep `setuptools-scm` in sync across CI --- .github/workflows/ci-cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 07816ed51..22acb66f4 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -309,6 +309,7 @@ jobs: pip install --user setuptools-scm + --constraint=requirements-build.txt shell: bash - name: Set the current dist version from Git if: steps.request-check.outputs.release-requested != 'true' From a7f9f5717ba9438312532af613f27c7cc465ba6e Mon Sep 17 00:00:00 2001 From: Jakub Jelen Date: Tue, 10 Sep 2024 13:55:06 +0200 Subject: [PATCH 3/5] Implement async SFTP from libssh 0.11 Signed-off-by: Jakub Jelen --- src/pylibsshext/includes/sftp.pxd | 16 +++ src/pylibsshext/sftp.pxd | 12 ++ src/pylibsshext/sftp.pyx | 226 ++++++++++++++++++++++-------- 3 files changed, 193 insertions(+), 61 deletions(-) diff --git a/src/pylibsshext/includes/sftp.pxd b/src/pylibsshext/includes/sftp.pxd index 94376578d..46a946478 100644 --- a/src/pylibsshext/includes/sftp.pxd +++ b/src/pylibsshext/includes/sftp.pxd @@ -84,6 +84,22 @@ cdef extern from "libssh/sftp.h" nogil: sftp_attributes sftp_stat(sftp_session session, const char *path) + struct sftp_aio_struct: + pass + ctypedef sftp_aio_struct * sftp_aio + ssize_t sftp_aio_begin_read(sftp_file file, size_t len, sftp_aio *aio) + ssize_t sftp_aio_wait_read(sftp_aio *aio, void *buf, size_t buf_size) + ssize_t sftp_aio_begin_write(sftp_file file, const void *buf, size_t len, sftp_aio *aio) + ssize_t sftp_aio_wait_write(sftp_aio *aio) + void sftp_aio_free(sftp_aio aio) + + struct sftp_limits_struct: + unsigned int max_packet_length + unsigned int max_read_length + unsigned int max_write_length + unsigned int max_open_handles + ctypedef sftp_limits_struct * sftp_limits_t + sftp_limits_t sftp_limits(sftp_session sftp) cdef extern from "sys/stat.h" nogil: cdef int S_IRWXU diff --git a/src/pylibsshext/sftp.pxd b/src/pylibsshext/sftp.pxd index 07f1afc6b..af355ae23 100644 --- a/src/pylibsshext/sftp.pxd +++ b/src/pylibsshext/sftp.pxd @@ -23,3 +23,15 @@ from pylibsshext.session cimport Session cdef class SFTP: cdef Session session cdef sftp.sftp_session _libssh_sftp_session + +cdef class SFTP_AIO: + cdef _aio_queue + cdef _remote_file + cdef _file_size + cdef _total_bytes_requested + cdef sftp.sftp_session _sftp + cdef sftp.sftp_limits_t _limits + cdef sftp.sftp_file _rf + +cdef class C_AIO: + cdef sftp.sftp_aio aio diff --git a/src/pylibsshext/sftp.pyx b/src/pylibsshext/sftp.pyx index 7528eba1e..bcc04f143 100644 --- a/src/pylibsshext/sftp.pyx +++ b/src/pylibsshext/sftp.pyx @@ -15,6 +15,9 @@ # License along with this library; if not, see file LICENSE.rst in this # repository. +import os +from collections import deque + from posix.fcntl cimport O_CREAT, O_RDONLY, O_TRUNC, O_WRONLY from cpython.bytes cimport PyBytes_AS_STRING @@ -58,85 +61,186 @@ cdef class SFTP: self._libssh_sftp_session = NULL def put(self, local_file, remote_file): + SFTP_AIO(self).put(local_file, remote_file) + + def get(self, remote_file, local_file): + SFTP_AIO(self).get(remote_file, local_file) + + def close(self): + if self._libssh_sftp_session is not NULL: + sftp.sftp_free(self._libssh_sftp_session) + self._libssh_sftp_session = NULL + + def _get_sftp_error_str(self): + error = sftp.sftp_get_error(self._libssh_sftp_session) + if error in MSG_MAP and error != sftp.SSH_FX_FAILURE: + return MSG_MAP[error] + return "Generic failure: %s" % self.session._get_session_error_str() + +cdef sftp.sftp_session get_sftp_session(SFTP sftp_obj): + return sftp_obj._libssh_sftp_session + +cdef class SFTP_AIO: + def __cinit__(self, SFTP sftp_obj): + self._sftp = get_sftp_session(sftp_obj) + + self._limits = sftp.sftp_limits(self._sftp) + if self._limits is NULL: + raise LibsshSFTPException("Failed to get remote SFTP limits [%s]" % (self._get_sftp_error_str())) + + def __init__(self, SFTP sftp_obj): + self._aio_queue = deque() + + def __dealloc__(self): + if self._rf is not NULL: + sftp.sftp_close(self._rf) + self._rf = NULL + + def put(self, local_file, remote_file): + # reset + self._aio_queue = deque() + self._total_bytes_requested = 0 + + cdef C_AIO aio cdef sftp.sftp_file rf - with open(local_file, "rb") as f: - remote_file_b = remote_file - if isinstance(remote_file_b, unicode): - remote_file_b = remote_file.encode("utf-8") + self._remote_file = remote_file - rf = sftp.sftp_open(self._libssh_sftp_session, remote_file_b, O_WRONLY | O_CREAT | O_TRUNC, sftp.S_IRWXU) - if rf is NULL: - raise LibsshSFTPException("Opening remote file [%s] for write failed with error [%s]" % (remote_file, self._get_sftp_error_str())) - buffer = f.read(SFTP_MAX_CHUNK) - - while buffer != b"": - length = len(buffer) - written = sftp.sftp_write(rf, PyBytes_AS_STRING(buffer), length) - if written != length: - sftp.sftp_close(rf) + remote_file_b = remote_file + if isinstance(remote_file_b, unicode): + remote_file_b = remote_file.encode("utf-8") + + rf = sftp.sftp_open(self._sftp, remote_file_b, O_WRONLY | O_CREAT | O_TRUNC, sftp.S_IRWXU) + if rf is NULL: + raise LibsshSFTPException("Opening remote file [%s] for write failed with error [%s]" % (remote_file, self._get_sftp_error_str())) + self._rf = rf + + with open(local_file, "rb") as f: + f.seek(0, os.SEEK_END) + self._file_size = f.tell() + f.seek(0, os.SEEK_SET) + + # start up to 10 requests before waiting for responses + i = 0 + while i < 10 and self._total_bytes_requested < self._file_size: + self._put_chunk(f) + i += 1 + + while len(self._aio_queue): + aio = self._aio_queue.popleft() + bytes_written = sftp.sftp_aio_wait_write(&aio.aio) + if bytes_written == libssh.SSH_ERROR: raise LibsshSFTPException( - "Writing to remote file [%s] failed with error [%s]" % ( - remote_file, - self._get_sftp_error_str(), - ) + "Failed to write to remote file [%s]: error [%s]" % (self._remote_file, self._get_sftp_error_str()) ) - buffer = f.read(SFTP_MAX_CHUNK) + # was freed in the wait if it did not fail + aio.aio = NULL + + # whole file read + if self._total_bytes_requested == self._file_size: + continue + + # else issue more read requests + self._put_chunk(f) + sftp.sftp_close(rf) + self._rf = NULL + + def _put_chunk(self, f): + to_write = min(self._file_size - self._total_bytes_requested, self._limits.max_write_length) + buffer = f.read(to_write) + if len(buffer) != to_write: + raise LibsshSFTPException("Read only [%d] but requested [%d] when reading from local file [%s] " % (len(buffer), to_write, self._remote_file)) + + cdef sftp.sftp_aio aio = NULL + bytes_requested = sftp.sftp_aio_begin_write(self._rf, PyBytes_AS_STRING(buffer), to_write, &aio) + if bytes_requested != to_write: + raise LibsshSFTPException("Failed to write chunk of size [%d] of file [%s] with error [%s]" + % (to_write, self._remote_file, self._get_sftp_error_str())) + self._total_bytes_requested += bytes_requested + c_aio = C_AIO() + c_aio.aio = aio + self._aio_queue.append(c_aio) def get(self, remote_file, local_file): - cdef sftp.sftp_file rf - cdef char *read_buffer = NULL + # reset + self._aio_queue = deque() + self._total_bytes_requested = 0 + + cdef C_AIO aio + cdef sftp.sftp_file rf = NULL cdef sftp.sftp_attributes attrs + cdef char *buffer = NULL + self._remote_file = remote_file remote_file_b = remote_file if isinstance(remote_file_b, unicode): remote_file_b = remote_file.encode("utf-8") - attrs = sftp.sftp_stat(self._libssh_sftp_session, remote_file_b) + attrs = sftp.sftp_stat(self._sftp, remote_file_b) if attrs is NULL: - raise LibsshSFTPException("Failed to stat the remote file [%s]. Error: [%s]" + raise LibsshSFTPException("Failed to stat the remote file [%s] with error [%s]" % (remote_file, self._get_sftp_error_str())) - file_size = attrs.size - - rf = sftp.sftp_open(self._libssh_sftp_session, remote_file_b, O_RDONLY, sftp.S_IRWXU) - if rf is NULL: - raise LibsshSFTPException("Opening remote file [%s] for read failed with error [%s]" % (remote_file, self._get_sftp_error_str())) + self._file_size = attrs.size + buffer_size = min(self._limits.max_read_length, self._file_size) try: + buffer = PyMem_Malloc(buffer_size) + + rf = sftp.sftp_open(self._sftp, remote_file_b, O_RDONLY, sftp.S_IRWXU) + if rf is NULL: + raise LibsshSFTPException("Opening remote file [%s] for reading failed with error [%s]" % (remote_file, self._get_sftp_error_str())) + self._rf = rf + with open(local_file, 'wb') as f: - buffer_size = min(SFTP_MAX_CHUNK, file_size) - read_buffer = PyMem_Malloc(buffer_size) - if read_buffer is NULL: - raise LibsshSFTPException("Memory allocation error") - - while True: - file_data = sftp.sftp_read(rf, read_buffer, sizeof(char) * buffer_size) - if file_data == 0: - break - elif file_data < 0: - sftp.sftp_close(rf) - raise LibsshSFTPException("Reading data from remote file [%s] failed with error [%s]" - % (remote_file, self._get_sftp_error_str())) - - bytes_written = f.write(read_buffer[:file_data]) - if bytes_written and file_data != bytes_written: - sftp.sftp_close(rf) - raise LibsshSFTPException("Number of bytes [%s] read from remote file [%s]" - " does not match number of bytes [%s] written to local file [%s]" - " due to error [%s]" - % (file_data, remote_file, bytes_written, local_file, self._get_sftp_error_str())) + # start up to 10 write requests before waiting for responses + i = 0 + while i < 10 and self._total_bytes_requested < self._file_size: + self._get_chunk() + i += 1 + + while len(self._aio_queue): + aio = self._aio_queue.popleft() + bytes_read = sftp.sftp_aio_wait_read(&aio.aio, buffer, buffer_size) + if bytes_read == libssh.SSH_ERROR: + raise LibsshSFTPException( + "Failed to read from remote file [%s]: error [%s]" % (self._remote_file, self._get_sftp_error_str()) + ) + # was freed in the wait if it did not fail -- otherwise the __dealloc__ will free it + aio.aio = NULL + + # write the file + f.write(buffer[:bytes_read]) + + # whole file read + if self._total_bytes_requested == self._file_size: + continue + + # else issue more read requests + self._get_chunk() + finally: - if read_buffer is not NULL: - PyMem_Free(read_buffer) - sftp.sftp_close(rf) + if buffer is not NULL: + PyMem_Free(buffer) + sftp.sftp_close(rf) + self._rf = NULL - def close(self): - if self._libssh_sftp_session is not NULL: - sftp.sftp_free(self._libssh_sftp_session) - self._libssh_sftp_session = NULL + def _get_chunk(self): + to_read = min(self._file_size - self._total_bytes_requested, self._limits.max_read_length) + cdef sftp.sftp_aio aio = NULL + bytes_requested = sftp.sftp_aio_begin_read(self._rf, to_read, &aio) + if bytes_requested != to_read: + raise LibsshSFTPException("Failed to request to read chunk of size [%d] of file [%s] with error [%s]" + % (to_read, self._remote_file, self._get_sftp_error_str())) + self._total_bytes_requested += bytes_requested + c_aio = C_AIO() + c_aio.aio = aio + self._aio_queue.append(c_aio) - def _get_sftp_error_str(self): - error = sftp.sftp_get_error(self._libssh_sftp_session) - if error in MSG_MAP and error != sftp.SSH_FX_FAILURE: - return MSG_MAP[error] - return "Generic failure: %s" % self.session._get_session_error_str() + +cdef class C_AIO: + def __cinit__(self): + self.aio = NULL + + def __dealloc__(self): + sftp.sftp_aio_free(self.aio) + self.aio = NULL From 8bc9d0a3aa7e25967ed1655c9e6bd2fd54615731 Mon Sep 17 00:00:00 2001 From: Jakub Jelen Date: Fri, 30 Aug 2024 11:20:00 +0200 Subject: [PATCH 4/5] tests: Reproducer for #341 Signed-off-by: Jakub Jelen --- tests/unit/sftp_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/sftp_test.py b/tests/unit/sftp_test.py index 59e08700f..8bfe877bd 100644 --- a/tests/unit/sftp_test.py +++ b/tests/unit/sftp_test.py @@ -23,8 +23,8 @@ def sftp_session(ssh_client_session): @pytest.fixture( - params=(32, SFTP_MAX_CHUNK + 1), - ids=('small-payload', 'large-payload'), + params=(32, SFTP_MAX_CHUNK + 1, 256 * 1024 - 1024 + 1), + ids=('small-payload', 'large-payload', 'huge-payload'), ) def transmit_payload(request: pytest.FixtureRequest) -> bytes: """Generate binary test payloads of assorted sizes. @@ -34,6 +34,9 @@ def transmit_payload(request: pytest.FixtureRequest) -> bytes: The choice SFTP_MAX_CHUNK + 1 (32kB + 1B) is meant to be 1B larger than the chunk size used in :file:`sftp.pyx` to make sure we excercise at least two rounds of reading/writing. + + The Choice 255kB + 1B is meant to be 1B larger than OpenSSH SFTP server supported + maximum reads and writes of 256 * 1024 - 1024 B per request. """ payload_len = request.param random_bytes = [ord(random.choice(string.printable)) for _ in range(payload_len)] From cd7f980e4ea45796060078a73efc78c92deb44c4 Mon Sep 17 00:00:00 2001 From: Jakub Jelen Date: Wed, 11 Sep 2024 16:54:40 +0200 Subject: [PATCH 5/5] Add changelog fragment Signed-off-by: Jakub Jelen --- docs/changelog-fragments/641.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changelog-fragments/641.feature.rst diff --git a/docs/changelog-fragments/641.feature.rst b/docs/changelog-fragments/641.feature.rst new file mode 100644 index 000000000..9f7663db8 --- /dev/null +++ b/docs/changelog-fragments/641.feature.rst @@ -0,0 +1 @@ +Added support for asynchronous SFTP file transfers to increase throughput -- by :user:`Jakuje`.