Add ability to download S3 files and list SFTP files

.github/workflows/deploy-production.yml

@@ -30,10 +30,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.13'
           cache: 'pip'

.github/workflows/deploy-qa.yml

@@ -30,10 +30,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.13'
           cache: 'pip'

.github/workflows/run-unit-tests.yml

@@ -9,16 +9,16 @@ jobs:
     name: Updates changelog
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: dangoslen/changelog-enforcer@v3
   test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.13'
           cache: 'pip'

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+## v1.12.3 9/18/26
+- Enable S3 client to download arbitrary files as BytesIO streams
+- Enable SFTP client to list available files
+
 ## v1.12.2 7/21/26
 - Enable S3 client to upload files in binary format if needed
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nypl_py_utils"
-version = "1.12.2"
+version = "1.12.3"
 authors = [
   { name="Aaron Friedman", email="aaronfriedman@nypl.org" },
 ]
@@ -26,30 +26,30 @@ dependencies = []
 avro-client = [
     "nypl_py_utils[log-helper]",
     "fastavro==1.12.2",
-    "requests==2.34.0"
+    "requests==2.34.2"
 ]
 azure-client = [
     "nypl_py_utils[log-helper]",
-    "mssql-python==1.9.0",
-    "pandas==3.0.3"
+    "mssql-python==1.15.0",
+    "pandas==3.0.6"
 ]
 cloudlibrary-client = [
     "nypl_py_utils[log-helper]",
-    "requests==2.34.0"
+    "requests==2.34.2"
 ]
 kinesis-client = [
     "nypl_py_utils[log-helper]",
-    "boto3==1.43.1",
-    "botocore==1.43.6"
+    "boto3==1.43.97",
+    "botocore==1.43.97"
 ]
 kms-client = [
     "nypl_py_utils[log-helper]",
-    "boto3==1.43.1",
-    "botocore==1.43.6"
+    "boto3==1.43.97",
+    "botocore==1.43.97"
 ]
 mysql-client = [
     "nypl_py_utils[log-helper]",
-    "mysql-connector-python==9.7.0"
+    "mysql-connector-python==26.7.0"
 ]
 oauth2-api-client = [
     "nypl_py_utils[log-helper]",
@@ -58,55 +58,55 @@ oauth2-api-client = [
 ]
 postgresql-client = [
     "nypl_py_utils[log-helper]",
-    "psycopg[binary]==3.3.4"
+    "psycopg[binary]==3.3.6"
 ]
 redshift-client = [
     "nypl_py_utils[log-helper]",
-    "botocore==1.43.6",
-    "redshift-connector==2.1.13"
+    "botocore==1.43.97",
+    "redshift-connector==2.1.16"
 ]
 s3-client = [
     "nypl_py_utils[log-helper]",
-    "boto3==1.43.1",
-    "botocore==1.43.6"
+    "boto3==1.43.97",
+    "botocore==1.43.97"
 ]
 secrets-manager-client = [
     "nypl_py_utils[log-helper]",
-    "boto3==1.43.1",
-    "botocore==1.43.6"
+    "boto3==1.43.97",
+    "botocore==1.43.97"
 ]
 sftp-client = [
     "nypl_py_utils[log-helper]",
-    "paramiko==4.0.0"
+    "paramiko==5.0.0"
 ]
 snowflake-client = [
     "nypl_py_utils[log-helper]",
-    "snowflake-connector-python==4.5.0"
+    "snowflake-connector-python==4.7.4"
 ]
 config-helper = [
     "nypl_py_utils[kms-client,log-helper]",
     "PyYAML==6.0.3"
 ]
 log-helper = [
-    "structlog==25.5.0"
+    "structlog==26.1.0"
 ]
 obfuscation-helper = [
     "nypl_py_utils[log-helper]",
     "bcrypt==5.0.0"
 ]
 patron-data-helper = [
     "nypl_py_utils[postgresql-client,redshift-client,log-helper]",
-    "pandas==3.0.3"
+    "pandas==3.0.6"
 ]
 research-catalog-identifier-helper = [
-    "requests==2.34.0"
+    "requests==2.34.2"
 ]
 development = [
     "nypl_py_utils[avro-client,azure-client,cloudlibrary-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,redshift-client,s3-client,secrets-manager-client,sftp-client,snowflake-client,config-helper,log-helper,obfuscation-helper,patron-data-helper,research-catalog-identifier-helper]",
     "flake8==7.3.0",
     "freezegun==1.5.5",
     "mock==5.2.0",
-    "pytest==9.0.3",
+    "pytest==9.1.1",
     "pytest-mock==3.15.1",
     "requests-mock==1.12.1"
 ]

src/nypl_py_utils/classes/s3_client.py

@@ -92,6 +92,30 @@ def upload_file(self, content, file_path):
             self.logger.error(error_msg)
             raise S3ClientError(error_msg) from None
 
+    def read_file(self, file_path):
+        """
+        Reads an arbitrary file from S3 as a BytesIO stream.
+
+        Parameters
+        ----------
+        file_path: str
+            The full path of the file that should be downloaded not including
+            the bucket. Example: "subdirectory/example_file.csv"
+        """
+        self.logger.info(f"Reading {file_path} in S3 bucket {self.bucket}")
+        try:
+            output_stream = BytesIO()
+            self.s3_client.download_fileobj(
+                self.bucket, file_path, output_stream
+            )
+            return output_stream
+        except ClientError as e:
+            error_msg = (
+                f"Error reading {file_path} in S3 bucket {self.bucket}: {e}"
+            )
+            self.logger.error(error_msg)
+            raise S3ClientError(error_msg) from None
+
 
 class S3ClientError(Exception):
     def __init__(self, message=None):

src/nypl_py_utils/classes/sftp_client.py

@@ -60,6 +60,17 @@ def download(self, remote_path, local_path):
             raise SftpClientError(
                 "Error downloading file: {}".format(e)) from None
 
+    def list_files(self, remote_path=None):
+        """Lists all files and directories in a remote directory"""
+        self.logger.info(f"Listing files in {remote_path}")
+        try:
+            return self.sftp_conn.listdir(remote_path)
+        except Exception as e:
+            self.logger.error("Error listing files: {}".format(e))
+            self.close_connection()
+            raise SftpClientError(
+                "Error listing files: {}".format(e)) from None
+
     def close_connection(self):
         """Closes the connection"""
         self.logger.debug("Closing connection to {}".format(self.host))

tests/test_s3_client.py

@@ -48,3 +48,14 @@ def test_upload_file_binary(self, test_instance):
         assert arguments[0].getvalue() == binary_content
         assert arguments[1] == "test_s3_bucket"
         assert arguments[2] == "test_filename.parquet"
+
+    def test_read_file(self, test_instance):
+        def mock_download(bucket, resource, stream):
+            assert bucket == "test_s3_bucket"
+            assert resource == "test_filename.txt"
+            stream.write("test_content".encode())
+
+        test_instance.s3_client.download_fileobj.side_effect = mock_download
+        assert test_instance.read_file(
+            "test_filename.txt"
+        ).getvalue().decode() == "test_content"

tests/test_sftp_client.py

@@ -89,6 +89,26 @@ def test_download_error(self, test_instance, mocker):
         test_instance.sftp_conn.close.assert_called_once()
         test_instance.ssh_client.close.assert_called_once()
 
+    def test_list_files(self, test_instance, mocker):
+        test_instance.sftp_conn = mocker.MagicMock()
+        test_instance.sftp_conn.listdir.return_value = ['filea', 'dirb']
+
+        assert test_instance.list_files('remote/path') == ['filea', 'dirb']
+
+        test_instance.sftp_conn.listdir.assert_called_once_with('remote/path')
+
+    def test_list_files_error(self, test_instance, mocker):
+        test_instance.ssh_client = mocker.MagicMock()
+        test_instance.sftp_conn = mocker.MagicMock()
+        test_instance.sftp_conn.listdir.side_effect = IOError('test error')
+
+        with pytest.raises(SftpClientError):
+            test_instance.list_files('remote/path')
+
+        test_instance.sftp_conn.listdir.assert_called_once_with('remote/path')
+        test_instance.sftp_conn.close.assert_called_once()
+        test_instance.ssh_client.close.assert_called_once()
+
     def test_close_connection(self, test_instance, mocker):
         test_instance.sftp_conn = mocker.MagicMock()
         test_instance.ssh_client = mocker.MagicMock()