diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 8dbba8f..96dfc87 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -30,10 +30,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python 3.13 - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.13' cache: 'pip' diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml index 319cfe3..72101a0 100644 --- a/.github/workflows/deploy-qa.yml +++ b/.github/workflows/deploy-qa.yml @@ -30,10 +30,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python 3.13 - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.13' cache: 'pip' diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 2440ae0..4ed78ea 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,16 +9,16 @@ jobs: name: Updates changelog runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: dangoslen/changelog-enforcer@v3 test: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python 3.13 - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.13' cache: 'pip' diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e8c1d..8dc0569 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## v1.12.3 9/18/26 +- Enable S3 client to download arbitrary files as BytesIO streams +- Enable SFTP client to list available files + ## v1.12.2 7/21/26 - Enable S3 client to upload files in binary format if needed diff --git a/pyproject.toml b/pyproject.toml index b92b5b6..82590df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "nypl_py_utils" -version = "1.12.2" +version = "1.12.3" authors = [ { name="Aaron Friedman", email="aaronfriedman@nypl.org" }, ] @@ -26,30 +26,30 @@ dependencies = [] avro-client = [ "nypl_py_utils[log-helper]", "fastavro==1.12.2", - "requests==2.34.0" + "requests==2.34.2" ] azure-client = [ "nypl_py_utils[log-helper]", - "mssql-python==1.9.0", - "pandas==3.0.3" + "mssql-python==1.15.0", + "pandas==3.0.6" ] cloudlibrary-client = [ "nypl_py_utils[log-helper]", - "requests==2.34.0" + "requests==2.34.2" ] kinesis-client = [ "nypl_py_utils[log-helper]", - "boto3==1.43.1", - "botocore==1.43.6" + "boto3==1.43.97", + "botocore==1.43.97" ] kms-client = [ "nypl_py_utils[log-helper]", - "boto3==1.43.1", - "botocore==1.43.6" + "boto3==1.43.97", + "botocore==1.43.97" ] mysql-client = [ "nypl_py_utils[log-helper]", - "mysql-connector-python==9.7.0" + "mysql-connector-python==26.7.0" ] oauth2-api-client = [ "nypl_py_utils[log-helper]", @@ -58,37 +58,37 @@ oauth2-api-client = [ ] postgresql-client = [ "nypl_py_utils[log-helper]", - "psycopg[binary]==3.3.4" + "psycopg[binary]==3.3.6" ] redshift-client = [ "nypl_py_utils[log-helper]", - "botocore==1.43.6", - "redshift-connector==2.1.13" + "botocore==1.43.97", + "redshift-connector==2.1.16" ] s3-client = [ "nypl_py_utils[log-helper]", - "boto3==1.43.1", - "botocore==1.43.6" + "boto3==1.43.97", + "botocore==1.43.97" ] secrets-manager-client = [ "nypl_py_utils[log-helper]", - "boto3==1.43.1", - "botocore==1.43.6" + "boto3==1.43.97", + "botocore==1.43.97" ] sftp-client = [ "nypl_py_utils[log-helper]", - "paramiko==4.0.0" + "paramiko==5.0.0" ] snowflake-client = [ "nypl_py_utils[log-helper]", - "snowflake-connector-python==4.5.0" + "snowflake-connector-python==4.7.4" ] config-helper = [ "nypl_py_utils[kms-client,log-helper]", "PyYAML==6.0.3" ] log-helper = [ - "structlog==25.5.0" + "structlog==26.1.0" ] obfuscation-helper = [ "nypl_py_utils[log-helper]", @@ -96,17 +96,17 @@ obfuscation-helper = [ ] patron-data-helper = [ "nypl_py_utils[postgresql-client,redshift-client,log-helper]", - "pandas==3.0.3" + "pandas==3.0.6" ] research-catalog-identifier-helper = [ - "requests==2.34.0" + "requests==2.34.2" ] development = [ "nypl_py_utils[avro-client,azure-client,cloudlibrary-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,redshift-client,s3-client,secrets-manager-client,sftp-client,snowflake-client,config-helper,log-helper,obfuscation-helper,patron-data-helper,research-catalog-identifier-helper]", "flake8==7.3.0", "freezegun==1.5.5", "mock==5.2.0", - "pytest==9.0.3", + "pytest==9.1.1", "pytest-mock==3.15.1", "requests-mock==1.12.1" ] diff --git a/src/nypl_py_utils/classes/s3_client.py b/src/nypl_py_utils/classes/s3_client.py index 9dab536..583963c 100644 --- a/src/nypl_py_utils/classes/s3_client.py +++ b/src/nypl_py_utils/classes/s3_client.py @@ -92,6 +92,30 @@ def upload_file(self, content, file_path): self.logger.error(error_msg) raise S3ClientError(error_msg) from None + def read_file(self, file_path): + """ + Reads an arbitrary file from S3 as a BytesIO stream. + + Parameters + ---------- + file_path: str + The full path of the file that should be downloaded not including + the bucket. Example: "subdirectory/example_file.csv" + """ + self.logger.info(f"Reading {file_path} in S3 bucket {self.bucket}") + try: + output_stream = BytesIO() + self.s3_client.download_fileobj( + self.bucket, file_path, output_stream + ) + return output_stream + except ClientError as e: + error_msg = ( + f"Error reading {file_path} in S3 bucket {self.bucket}: {e}" + ) + self.logger.error(error_msg) + raise S3ClientError(error_msg) from None + class S3ClientError(Exception): def __init__(self, message=None): diff --git a/src/nypl_py_utils/classes/sftp_client.py b/src/nypl_py_utils/classes/sftp_client.py index 1e01ec7..ce8645b 100644 --- a/src/nypl_py_utils/classes/sftp_client.py +++ b/src/nypl_py_utils/classes/sftp_client.py @@ -60,6 +60,17 @@ def download(self, remote_path, local_path): raise SftpClientError( "Error downloading file: {}".format(e)) from None + def list_files(self, remote_path=None): + """Lists all files and directories in a remote directory""" + self.logger.info(f"Listing files in {remote_path}") + try: + return self.sftp_conn.listdir(remote_path) + except Exception as e: + self.logger.error("Error listing files: {}".format(e)) + self.close_connection() + raise SftpClientError( + "Error listing files: {}".format(e)) from None + def close_connection(self): """Closes the connection""" self.logger.debug("Closing connection to {}".format(self.host)) diff --git a/tests/test_s3_client.py b/tests/test_s3_client.py index d82541f..8b4b232 100644 --- a/tests/test_s3_client.py +++ b/tests/test_s3_client.py @@ -48,3 +48,14 @@ def test_upload_file_binary(self, test_instance): assert arguments[0].getvalue() == binary_content assert arguments[1] == "test_s3_bucket" assert arguments[2] == "test_filename.parquet" + + def test_read_file(self, test_instance): + def mock_download(bucket, resource, stream): + assert bucket == "test_s3_bucket" + assert resource == "test_filename.txt" + stream.write("test_content".encode()) + + test_instance.s3_client.download_fileobj.side_effect = mock_download + assert test_instance.read_file( + "test_filename.txt" + ).getvalue().decode() == "test_content" diff --git a/tests/test_sftp_client.py b/tests/test_sftp_client.py index 0172145..ac6d5dd 100644 --- a/tests/test_sftp_client.py +++ b/tests/test_sftp_client.py @@ -89,6 +89,26 @@ def test_download_error(self, test_instance, mocker): test_instance.sftp_conn.close.assert_called_once() test_instance.ssh_client.close.assert_called_once() + def test_list_files(self, test_instance, mocker): + test_instance.sftp_conn = mocker.MagicMock() + test_instance.sftp_conn.listdir.return_value = ['filea', 'dirb'] + + assert test_instance.list_files('remote/path') == ['filea', 'dirb'] + + test_instance.sftp_conn.listdir.assert_called_once_with('remote/path') + + def test_list_files_error(self, test_instance, mocker): + test_instance.ssh_client = mocker.MagicMock() + test_instance.sftp_conn = mocker.MagicMock() + test_instance.sftp_conn.listdir.side_effect = IOError('test error') + + with pytest.raises(SftpClientError): + test_instance.list_files('remote/path') + + test_instance.sftp_conn.listdir.assert_called_once_with('remote/path') + test_instance.sftp_conn.close.assert_called_once() + test_instance.ssh_client.close.assert_called_once() + def test_close_connection(self, test_instance, mocker): test_instance.sftp_conn = mocker.MagicMock() test_instance.ssh_client = mocker.MagicMock()