From 65a3b61201e883a1f7ea6369ef22a2d067eca2be Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 13 Nov 2024 19:42:39 +0000 Subject: [PATCH 1/6] moves add_to_db to functions module --- aim/cli/digifeeds.py | 7 +++-- .../{check_zephir.py => functions.py} | 0 docs/api/aim.digifeeds.check_zephir.rst | 7 ----- docs/api/aim.digifeeds.functions.rst | 7 +++++ docs/api/aim.digifeeds.rst | 2 +- tests/digifeeds/test_check_zephir.py | 30 ++++++++++--------- 6 files changed, 29 insertions(+), 24 deletions(-) rename aim/digifeeds/{check_zephir.py => functions.py} (100%) delete mode 100644 docs/api/aim.digifeeds.check_zephir.rst create mode 100644 docs/api/aim.digifeeds.functions.rst diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index 2bbffdc..3a2f557 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -6,9 +6,12 @@ from typing_extensions import Annotated from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket -from aim.digifeeds.check_zephir import check_zephir as check_zephir_for_barcode + +# from aim.digifeeds.check_zephir import check_zephir as check_zephir_for_barcode from aim.digifeeds.move_to_pickup import move_to_pickup as move_volume_to_pickup from aim.digifeeds.database import models, main +from aim.digifeeds import functions + import json import sys @@ -57,7 +60,7 @@ def check_zephir( """ print(f"Checking Zephir for {barcode}") - item = check_zephir_for_barcode(barcode) + item = functions.check_zephir(barcode) if item: print(f"{barcode} is in Zephir") else: diff --git a/aim/digifeeds/check_zephir.py b/aim/digifeeds/functions.py similarity index 100% rename from aim/digifeeds/check_zephir.py rename to aim/digifeeds/functions.py diff --git a/docs/api/aim.digifeeds.check_zephir.rst b/docs/api/aim.digifeeds.check_zephir.rst deleted file mode 100644 index 5beaa2a..0000000 --- a/docs/api/aim.digifeeds.check_zephir.rst +++ /dev/null @@ -1,7 +0,0 @@ -aim.digifeeds.check\_zephir module -================================== - -.. automodule:: aim.digifeeds.check_zephir - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/aim.digifeeds.functions.rst b/docs/api/aim.digifeeds.functions.rst new file mode 100644 index 0000000..ea51545 --- /dev/null +++ b/docs/api/aim.digifeeds.functions.rst @@ -0,0 +1,7 @@ +aim.digifeeds.functions module +============================== + +.. automodule:: aim.digifeeds.functions + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/aim.digifeeds.rst b/docs/api/aim.digifeeds.rst index 9380920..ce7c0a8 100644 --- a/docs/api/aim.digifeeds.rst +++ b/docs/api/aim.digifeeds.rst @@ -22,8 +22,8 @@ Submodules aim.digifeeds.add_to_db aim.digifeeds.alma_client - aim.digifeeds.check_zephir aim.digifeeds.db_client + aim.digifeeds.functions aim.digifeeds.item aim.digifeeds.list_barcodes_in_bucket aim.digifeeds.move_to_pickup diff --git a/tests/digifeeds/test_check_zephir.py b/tests/digifeeds/test_check_zephir.py index 2ba3fbd..55d98bc 100644 --- a/tests/digifeeds/test_check_zephir.py +++ b/tests/digifeeds/test_check_zephir.py @@ -2,7 +2,8 @@ import responses import json from aim.services import S -from aim.digifeeds.check_zephir import check_zephir +from aim.digifeeds import functions +from aim.digifeeds.functions import check_zephir @pytest.fixture @@ -19,11 +20,11 @@ def barcode(): @responses.activate def test_barcode_is_in_zephir(mocker, item_data, barcode): - get_item_mock = mocker.patch( - "aim.digifeeds.check_zephir.DBClient.get_item", return_value=item_data + get_item_mock = mocker.patch.object( + functions.DBClient, "get_item", return_value=item_data ) - add_status_mock = mocker.patch( - "aim.digifeeds.check_zephir.DBClient.add_item_status", return_value=item_data + add_status_mock = mocker.patch.object( + functions.DBClient, "add_item_status", return_value=item_data ) responses.get(f"{S.zephir_bib_api_url}/mdp.{barcode}", json={}, status=200) @@ -35,12 +36,12 @@ def test_barcode_is_in_zephir(mocker, item_data, barcode): def test_barcode_already_has_in_zephir_status(mocker, item_data, barcode): item_data["statuses"][0]["name"] = "in_zephir" - get_item_mock = mocker.patch( - "aim.digifeeds.check_zephir.DBClient.get_item", return_value=item_data + get_item_mock = mocker.patch.object( + functions.DBClient, "get_item", return_value=item_data ) - add_status_mock = mocker.patch( - "aim.digifeeds.check_zephir.DBClient.add_item_status", return_value=item_data + add_status_mock = mocker.patch.object( + functions.DBClient, "add_item_status", return_value=item_data ) result = check_zephir(barcode) @@ -51,11 +52,12 @@ def test_barcode_already_has_in_zephir_status(mocker, item_data, barcode): @responses.activate def test_barcode_not_in_zephir(mocker, item_data, barcode): - get_item_mock = mocker.patch( - "aim.digifeeds.check_zephir.DBClient.get_item", return_value=item_data + get_item_mock = mocker.patch.object( + functions.DBClient, "get_item", return_value=item_data ) - add_status_mock = mocker.patch( - "aim.digifeeds.check_zephir.DBClient.add_item_status", return_value=item_data + + add_status_mock = mocker.patch.object( + functions.DBClient, "add_item_status", return_value=item_data ) responses.get(f"{S.zephir_bib_api_url}/mdp.{barcode}", json={}, status=404) @@ -66,7 +68,7 @@ def test_barcode_not_in_zephir(mocker, item_data, barcode): def test_barcdoe_is_not_in_db(mocker, barcode): - mocker.patch("aim.digifeeds.check_zephir.DBClient.get_item", return_value=None) + mocker.patch.object(functions.DBClient, "get_item", return_value=None) with pytest.raises(Exception) as exc_info: check_zephir(barcode) assert exc_info.type is Exception From b7c54bf5750459a61fe1e6fa70e7e0ebed034755 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 13 Nov 2024 19:53:52 +0000 Subject: [PATCH 2/6] moves list barcodes to functions module --- aim/cli/digifeeds.py | 5 +---- aim/digifeeds/functions.py | 14 ++++++++++++++ aim/digifeeds/list_barcodes_in_bucket.py | 15 --------------- .../api/aim.digifeeds.list_barcodes_in_bucket.rst | 7 ------- docs/api/aim.digifeeds.rst | 1 - tests/cli/test_digifeeds.py | 6 ++++-- tests/digifeeds/test_list_barcodes_in_bucket.py | 4 ++-- 7 files changed, 21 insertions(+), 31 deletions(-) delete mode 100644 aim/digifeeds/list_barcodes_in_bucket.py delete mode 100644 docs/api/aim.digifeeds.list_barcodes_in_bucket.rst diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index 3a2f557..b5466ae 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -5,9 +5,6 @@ import typer from typing_extensions import Annotated from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db -from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket - -# from aim.digifeeds.check_zephir import check_zephir as check_zephir_for_barcode from aim.digifeeds.move_to_pickup import move_to_pickup as move_volume_to_pickup from aim.digifeeds.database import models, main from aim.digifeeds import functions @@ -81,7 +78,7 @@ def list_barcodes_in_input_bucket(): """ List the barcodes currently in the input directory in the S3 bucket. """ - json.dump(list_barcodes_in_bucket(), sys.stdout) + json.dump(functions.list_barcodes_in_input_bucket(), sys.stdout) @app.command() diff --git a/aim/digifeeds/functions.py b/aim/digifeeds/functions.py index 2b93fa8..2400bb8 100644 --- a/aim/digifeeds/functions.py +++ b/aim/digifeeds/functions.py @@ -2,6 +2,20 @@ from aim.digifeeds.db_client import DBClient from aim.digifeeds.item import Item import requests +import boto3 +from pathlib import Path + + +def list_barcodes_in_input_bucket(): + s3 = boto3.client( + "s3", + aws_access_key_id=S.digifeeds_s3_access_key, + aws_secret_access_key=S.digifeeds_s3_secret_access_key, + ) + prefix = S.digifeeds_s3_input_path + "/" + response = s3.list_objects_v2(Bucket=S.digifeeds_s3_bucket, Prefix=prefix) + barcodes = [Path(object["Key"]).stem for object in response["Contents"]] + return barcodes def check_zephir(barcode: str): diff --git a/aim/digifeeds/list_barcodes_in_bucket.py b/aim/digifeeds/list_barcodes_in_bucket.py deleted file mode 100644 index 18002ce..0000000 --- a/aim/digifeeds/list_barcodes_in_bucket.py +++ /dev/null @@ -1,15 +0,0 @@ -import boto3 -from aim.services import S -from pathlib import Path - - -def list_barcodes_in_bucket(): - s3 = boto3.client( - "s3", - aws_access_key_id=S.digifeeds_s3_access_key, - aws_secret_access_key=S.digifeeds_s3_secret_access_key, - ) - prefix = S.digifeeds_s3_input_path + "/" - response = s3.list_objects_v2(Bucket=S.digifeeds_s3_bucket, Prefix=prefix) - barcodes = [Path(object["Key"]).stem for object in response["Contents"]] - return barcodes diff --git a/docs/api/aim.digifeeds.list_barcodes_in_bucket.rst b/docs/api/aim.digifeeds.list_barcodes_in_bucket.rst deleted file mode 100644 index 93a2f66..0000000 --- a/docs/api/aim.digifeeds.list_barcodes_in_bucket.rst +++ /dev/null @@ -1,7 +0,0 @@ -aim.digifeeds.list\_barcodes\_in\_bucket module -=============================================== - -.. automodule:: aim.digifeeds.list_barcodes_in_bucket - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/aim.digifeeds.rst b/docs/api/aim.digifeeds.rst index ce7c0a8..f0fc64b 100644 --- a/docs/api/aim.digifeeds.rst +++ b/docs/api/aim.digifeeds.rst @@ -25,5 +25,4 @@ Submodules aim.digifeeds.db_client aim.digifeeds.functions aim.digifeeds.item - aim.digifeeds.list_barcodes_in_bucket aim.digifeeds.move_to_pickup diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index 4e729cc..3e61f4b 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -6,6 +6,7 @@ from aim.cli.main import app from aim.services import S from aim.digifeeds.item import Item +from aim.digifeeds import functions import aim.cli.digifeeds as digifeeds_cli runner = CliRunner() @@ -78,8 +79,9 @@ def test_load_statuses(mocker): def test_list_barcodes_in_input_bucket(mocker): - list_barcodes_mock = mocker.patch( - "aim.cli.digifeeds.list_barcodes_in_bucket", + list_barcodes_mock = mocker.patch.object( + functions, + "list_barcodes_in_input_bucket", return_value=["barcode1", "barcode2"], ) result = runner.invoke(app, ["digifeeds", "list-barcodes-in-input-bucket"]) diff --git a/tests/digifeeds/test_list_barcodes_in_bucket.py b/tests/digifeeds/test_list_barcodes_in_bucket.py index f44fb8f..2323d39 100644 --- a/tests/digifeeds/test_list_barcodes_in_bucket.py +++ b/tests/digifeeds/test_list_barcodes_in_bucket.py @@ -1,7 +1,7 @@ import boto3 from moto import mock_aws from aim.services import S -from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket +from aim.digifeeds.functions import list_barcodes_in_input_bucket @mock_aws @@ -22,5 +22,5 @@ def test_list_barcodes_in_bucket(): ) barcode2.put(Body="some text") - result = list_barcodes_in_bucket() + result = list_barcodes_in_input_bucket() assert result == ["barcode1", "barcode2"] From 3aac3268e8d841f369b2c0af3d8109db3383735a Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 13 Nov 2024 20:00:55 +0000 Subject: [PATCH 3/6] moves move_to_pickup to functions --- aim/cli/digifeeds.py | 3 +-- aim/digifeeds/functions.py | 30 +++++++++++++++++++++ aim/digifeeds/move_to_pickup.py | 33 ----------------------- docs/api/aim.digifeeds.move_to_pickup.rst | 7 ----- docs/api/aim.digifeeds.rst | 1 - tests/cli/test_digifeeds.py | 5 ++-- tests/digifeeds/test_move_to_pickup.py | 2 +- 7 files changed, 34 insertions(+), 47 deletions(-) delete mode 100644 aim/digifeeds/move_to_pickup.py delete mode 100644 docs/api/aim.digifeeds.move_to_pickup.rst diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index b5466ae..a654ce0 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -5,7 +5,6 @@ import typer from typing_extensions import Annotated from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db -from aim.digifeeds.move_to_pickup import move_to_pickup as move_volume_to_pickup from aim.digifeeds.database import models, main from aim.digifeeds import functions @@ -94,7 +93,7 @@ def move_to_pickup( folder in the bucket and prefixed with the date and time. """ print(f'Moving barcode "{barcode}" from the s3 bucket to the google drive') - item = move_volume_to_pickup(barcode) + item = functions.move_to_pickup(barcode) if item is None: print("Item has not been in zephir long enough") else: diff --git a/aim/digifeeds/functions.py b/aim/digifeeds/functions.py index 2400bb8..2b8824a 100644 --- a/aim/digifeeds/functions.py +++ b/aim/digifeeds/functions.py @@ -4,6 +4,36 @@ import requests import boto3 from pathlib import Path +from rclone_python import rclone +from datetime import datetime + + +def move_to_pickup(barcode: str): + raw_item = DBClient().get_item(barcode) + if raw_item is None: + raise Exception("Item not found in database") + + item = Item(raw_item) + + if not item.in_zephir_for_long_enough: + return None + + DBClient().add_item_status(barcode=barcode, status="copying_start") + rclone.copyto( + f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}/{barcode}.zip", + f"{S.digifeeds_gdrive_rclone_remote}:{barcode}.zip", + ) + DBClient().add_item_status(barcode=barcode, status="copying_end") + timestamp = datetime.now().strftime("%F_%H-%M-%S") + rclone.moveto( + f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}/{barcode}.zip", + f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_processed_path}/{timestamp}_{barcode}.zip", + ) + final_raw_item = DBClient().add_item_status( + barcode=barcode, status="pending_deletion" + ) + + return final_raw_item def list_barcodes_in_input_bucket(): diff --git a/aim/digifeeds/move_to_pickup.py b/aim/digifeeds/move_to_pickup.py deleted file mode 100644 index d3a6376..0000000 --- a/aim/digifeeds/move_to_pickup.py +++ /dev/null @@ -1,33 +0,0 @@ -from rclone_python import rclone -from aim.digifeeds.item import Item -from aim.digifeeds.db_client import DBClient -from aim.services import S -from datetime import datetime - - -def move_to_pickup(barcode: str): - raw_item = DBClient().get_item(barcode) - if raw_item is None: - raise Exception("Item not found in database") - - item = Item(raw_item) - - if not item.in_zephir_for_long_enough: - return None - - DBClient().add_item_status(barcode=barcode, status="copying_start") - rclone.copyto( - f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}/{barcode}.zip", - f"{S.digifeeds_gdrive_rclone_remote}:{barcode}.zip", - ) - DBClient().add_item_status(barcode=barcode, status="copying_end") - timestamp = datetime.now().strftime("%F_%H-%M-%S") - rclone.moveto( - f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}/{barcode}.zip", - f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_processed_path}/{timestamp}_{barcode}.zip", - ) - final_raw_item = DBClient().add_item_status( - barcode=barcode, status="pending_deletion" - ) - - return final_raw_item diff --git a/docs/api/aim.digifeeds.move_to_pickup.rst b/docs/api/aim.digifeeds.move_to_pickup.rst deleted file mode 100644 index 8a5ccbd..0000000 --- a/docs/api/aim.digifeeds.move_to_pickup.rst +++ /dev/null @@ -1,7 +0,0 @@ -aim.digifeeds.move\_to\_pickup module -===================================== - -.. automodule:: aim.digifeeds.move_to_pickup - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/aim.digifeeds.rst b/docs/api/aim.digifeeds.rst index f0fc64b..6d83a5b 100644 --- a/docs/api/aim.digifeeds.rst +++ b/docs/api/aim.digifeeds.rst @@ -25,4 +25,3 @@ Submodules aim.digifeeds.db_client aim.digifeeds.functions aim.digifeeds.item - aim.digifeeds.move_to_pickup diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index 3e61f4b..27dcadc 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -7,7 +7,6 @@ from aim.services import S from aim.digifeeds.item import Item from aim.digifeeds import functions -import aim.cli.digifeeds as digifeeds_cli runner = CliRunner() @@ -121,7 +120,7 @@ def test_check_zephir_for_item_when_item_is_not_in_zephir(item_data): def test_move_to_pickup_success(mocker, item_data): item = Item(item_data) move_volume_to_pickup_mock = mocker.patch.object( - digifeeds_cli, "move_volume_to_pickup", return_value=item + functions, "move_to_pickup", return_value=item ) result = runner.invoke(app, ["digifeeds", "move-to-pickup", "some_barcode"]) @@ -133,7 +132,7 @@ def test_move_to_pickup_success(mocker, item_data): def test_move_to_pickup_where_not_in_zephir(mocker): move_volume_to_pickup_mock = mocker.patch.object( - digifeeds_cli, "move_volume_to_pickup", return_value=None + functions, "move_to_pickup", return_value=None ) result = runner.invoke(app, ["digifeeds", "move-to-pickup", "some_barcode"]) diff --git a/tests/digifeeds/test_move_to_pickup.py b/tests/digifeeds/test_move_to_pickup.py index 5b69657..a7b2ad9 100644 --- a/tests/digifeeds/test_move_to_pickup.py +++ b/tests/digifeeds/test_move_to_pickup.py @@ -1,4 +1,4 @@ -from aim.digifeeds.move_to_pickup import move_to_pickup, rclone, DBClient +from aim.digifeeds.functions import move_to_pickup, rclone, DBClient import json import pytest from datetime import datetime From 3ab278c2ef3d9b7f3e408c8d0ea73917e85de950 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 13 Nov 2024 20:13:34 +0000 Subject: [PATCH 4/6] moves add_to_db to functions --- aim/cli/digifeeds.py | 3 +- aim/digifeeds/add_to_db.py | 43 ---------------------------- aim/digifeeds/functions.py | 41 ++++++++++++++++++++++++++ docs/api/aim.digifeeds.add_to_db.rst | 7 ----- docs/api/aim.digifeeds.rst | 1 - tests/cli/test_digifeeds.py | 2 +- tests/digifeeds/test_add_to_db.py | 43 +++++++++++++++------------- 7 files changed, 66 insertions(+), 74 deletions(-) delete mode 100644 aim/digifeeds/add_to_db.py delete mode 100644 docs/api/aim.digifeeds.add_to_db.rst diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index a654ce0..d91b3cc 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -4,7 +4,6 @@ import typer from typing_extensions import Annotated -from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db from aim.digifeeds.database import models, main from aim.digifeeds import functions @@ -33,7 +32,7 @@ def add_to_db( barcode (str): Barcode of item """ print(f'Adding barcode "{barcode}" to database') - item = add_to_digifeeds_db(barcode) + item = functions.add_to_db(barcode) if item.has_status("not_found_in_alma"): print("Item not found in alma.") if item.has_status("added_to_digifeeds_set"): diff --git a/aim/digifeeds/add_to_db.py b/aim/digifeeds/add_to_db.py deleted file mode 100644 index f411c58..0000000 --- a/aim/digifeeds/add_to_db.py +++ /dev/null @@ -1,43 +0,0 @@ -from aim.digifeeds.alma_client import AlmaClient -from aim.digifeeds.db_client import DBClient -from aim.digifeeds.item import Item -from requests.exceptions import HTTPError - - -def add_to_db(barcode: str): - """Add a barcode to the digifeeds database - - Args: - barcode (str): Barcode of the item - - Raises: - ext_inst: HTTPError - - Returns: - aim.digifeeds.database.models.Item: Item object - """ - item = Item(DBClient().get_or_add_item(barcode)) - if not item.has_status("added_to_digifeeds_set"): - try: - AlmaClient().add_barcode_to_digifeeds_set(barcode) - except HTTPError as ext_inst: - errorList = ext_inst.response.json()["errorList"]["error"] - if any(e["errorCode"] == "60120" for e in errorList): - if not item.has_status("not_found_in_alma"): - item = Item( - DBClient().add_item_status( - barcode=barcode, status="not_found_in_alma" - ) - ) - return item - elif any(e["errorCode"] == "60115" for e in errorList): - # 60115 means the barcode is already in the set. That means the - # db entry from this barcdoe needs to have - # added_to_digifeeds_set - pass - else: - raise ext_inst - item = Item( - DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set") - ) - return item diff --git a/aim/digifeeds/functions.py b/aim/digifeeds/functions.py index 2b8824a..5c0dcee 100644 --- a/aim/digifeeds/functions.py +++ b/aim/digifeeds/functions.py @@ -6,6 +6,47 @@ from pathlib import Path from rclone_python import rclone from datetime import datetime +from aim.digifeeds.alma_client import AlmaClient +from requests.exceptions import HTTPError + + +def add_to_db(barcode: str): + """Add a barcode to the digifeeds database + + Args: + barcode (str): Barcode of the item + + Raises: + ext_inst: HTTPError + + Returns: + aim.digifeeds.database.models.Item: Item object + """ + item = Item(DBClient().get_or_add_item(barcode)) + if not item.has_status("added_to_digifeeds_set"): + try: + AlmaClient().add_barcode_to_digifeeds_set(barcode) + except HTTPError as ext_inst: + errorList = ext_inst.response.json()["errorList"]["error"] + if any(e["errorCode"] == "60120" for e in errorList): + if not item.has_status("not_found_in_alma"): + item = Item( + DBClient().add_item_status( + barcode=barcode, status="not_found_in_alma" + ) + ) + return item + elif any(e["errorCode"] == "60115" for e in errorList): + # 60115 means the barcode is already in the set. That means the + # db entry from this barcdoe needs to have + # added_to_digifeeds_set + pass + else: + raise ext_inst + item = Item( + DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set") + ) + return item def move_to_pickup(barcode: str): diff --git a/docs/api/aim.digifeeds.add_to_db.rst b/docs/api/aim.digifeeds.add_to_db.rst deleted file mode 100644 index 8b8406c..0000000 --- a/docs/api/aim.digifeeds.add_to_db.rst +++ /dev/null @@ -1,7 +0,0 @@ -aim.digifeeds.add\_to\_db module -================================ - -.. automodule:: aim.digifeeds.add_to_db - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/aim.digifeeds.rst b/docs/api/aim.digifeeds.rst index 6d83a5b..a56a99b 100644 --- a/docs/api/aim.digifeeds.rst +++ b/docs/api/aim.digifeeds.rst @@ -20,7 +20,6 @@ Submodules .. toctree:: :maxdepth: 4 - aim.digifeeds.add_to_db aim.digifeeds.alma_client aim.digifeeds.db_client aim.digifeeds.functions diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index 27dcadc..5d72460 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -61,7 +61,7 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): def test_add_to_db_where_item_is_not_in_alma(item_data, mocker): item_data["statuses"][0]["name"] = "not_found_in_alma" item = Item(item_data) - mocker.patch("aim.cli.digifeeds.add_to_digifeeds_db", return_value=item) + mocker.patch.object(functions, "add_to_db", return_value=item) result = runner.invoke(app, ["digifeeds", "add-to-db", "some_barcode"]) assert "Item not found in alma" in result.stdout diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py index f834292..669f99e 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_db.py @@ -3,7 +3,9 @@ import json import pytest import responses -from aim.digifeeds.add_to_db import add_to_db +from aim.digifeeds.functions import add_to_db +from aim.digifeeds.alma_client import AlmaClient +from aim.digifeeds.db_client import DBClient from requests.exceptions import HTTPError from aim.services import S @@ -16,8 +18,8 @@ def item_data(): def test_add_to_db_barcode_thats_in_the_digifeeds_set(mocker, item_data): - get_item_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + get_item_mock = mocker.patch.object( + DBClient, "get_or_add_item", return_value=item_data ) result = add_to_db("my_barcode") get_item_mock.assert_called_once() @@ -29,11 +31,11 @@ def test_add_to_db_barcode_thats_in_the_digifeeds_set_but_doesnt_have_status( mocker, item_data ): item_data["statuses"][0]["name"] = "some_other_status" - get_item_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + get_item_mock = mocker.patch.object( + DBClient, "get_or_add_item", return_value=item_data ) - add_status_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.add_item_status", return_value=item_data + add_status_mock = mocker.patch.object( + DBClient, "add_item_status", return_value=item_data ) error_body = { "errorsExist": True, @@ -65,14 +67,15 @@ def test_add_to_db_barcode_thats_in_the_digifeeds_set_but_doesnt_have_status( def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" - get_item_mock = mocker.patch( - "aim.digifeeds.add_to_db.DBClient.get_or_add_item", return_value=item_data + get_item_mock = mocker.patch.object( + DBClient, "get_or_add_item", return_value=item_data ) - add_status_mock = mocker.patch( - "aim.digifeeds.add_to_db.DBClient.add_item_status", return_value=item_data + add_status_mock = mocker.patch.object( + DBClient, "add_item_status", return_value=item_data ) - add_to_digifeeds_set_mock = mocker.patch( - "aim.digifeeds.add_to_db.AlmaClient.add_barcode_to_digifeeds_set", + add_to_digifeeds_set_mock = mocker.patch.object( + AlmaClient, + "add_barcode_to_digifeeds_set", return_value=item_data, ) result = add_to_db("some_barcode") @@ -85,11 +88,11 @@ def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): @responses.activate def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" - get_item_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + get_item_mock = mocker.patch.object( + DBClient, "get_or_add_item", return_value=item_data ) - add_status_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.add_item_status", return_value=item_data + add_status_mock = mocker.patch.object( + DBClient, "add_item_status", return_value=item_data ) error_body = { "errorsExist": True, @@ -122,10 +125,10 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): @responses.activate def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" - get_item_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + get_item_mock = mocker.patch.object( + DBClient, "get_or_add_item", return_value=item_data ) - add_status_mock = mocker.patch("aim.digifeeds.db_client.DBClient.add_item_status") + add_status_mock = mocker.patch.object(DBClient, "add_item_status") error_body = { "errorsExist": True, "errorList": { From 55a21ae38de65e1dfd410bac15419b7457872962 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 13 Nov 2024 20:31:08 +0000 Subject: [PATCH 5/6] renames add_to_db to add_to_digifeeds_set --- aim/cli/digifeeds.py | 4 +-- aim/digifeeds/functions.py | 8 +++-- tests/cli/test_digifeeds.py | 10 +++--- ..._to_db.py => test_add_to_digifeeds_set.py} | 34 ++++++++++--------- 4 files changed, 31 insertions(+), 25 deletions(-) rename tests/digifeeds/{test_add_to_db.py => test_add_to_digifeeds_set.py} (82%) diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index d91b3cc..0bba685 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -15,7 +15,7 @@ @app.command() -def add_to_db( +def add_to_digifeeds_set( barcode: Annotated[ str, typer.Argument(help="The barcode to be added to the database"), @@ -32,7 +32,7 @@ def add_to_db( barcode (str): Barcode of item """ print(f'Adding barcode "{barcode}" to database') - item = functions.add_to_db(barcode) + item = functions.add_to_digifeeds_set(barcode) if item.has_status("not_found_in_alma"): print("Item not found in alma.") if item.has_status("added_to_digifeeds_set"): diff --git a/aim/digifeeds/functions.py b/aim/digifeeds/functions.py index 5c0dcee..d206b2f 100644 --- a/aim/digifeeds/functions.py +++ b/aim/digifeeds/functions.py @@ -10,8 +10,12 @@ from requests.exceptions import HTTPError -def add_to_db(barcode: str): - """Add a barcode to the digifeeds database +def add_to_digifeeds_set(barcode: str): + """ + Adds a barcode to the digifeeds set and to the digifeeds database if it + doesn't already exist. If the barcode is in the digifeeds set already but + doesn't have an added_to_digifeeds_set status in the digifeeds db, that + status gets added. Args: barcode (str): Barcode of the item diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index 5d72460..21a4053 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -38,7 +38,7 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): "fail_on_invalid_id": "true", } add_to_digifeeds_body = {"members": {"member": [{"id": "some_barcode"}]}} - add_to_digifeeds_set = responses.post( + add_to_digifeeds_set_stub = responses.post( add_to_digifeeds_url, match=[ matchers.query_param_matcher(add_to_digifeeds_query), @@ -48,10 +48,10 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): status=200, ) - result = runner.invoke(app, ["digifeeds", "add-to-db", "some_barcode"]) + result = runner.invoke(app, ["digifeeds", "add-to-digifeeds-set", "some_barcode"]) assert get_item.call_count == 1 assert post_item.call_count == 1 - assert add_to_digifeeds_set.call_count == 1 + assert add_to_digifeeds_set_stub.call_count == 1 assert add_item_status.call_count == 1 assert result.exit_code == 0 assert 'Adding barcode "some_barcode" to database' in result.stdout @@ -61,9 +61,9 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): def test_add_to_db_where_item_is_not_in_alma(item_data, mocker): item_data["statuses"][0]["name"] = "not_found_in_alma" item = Item(item_data) - mocker.patch.object(functions, "add_to_db", return_value=item) + mocker.patch.object(functions, "add_to_digifeeds_set", return_value=item) - result = runner.invoke(app, ["digifeeds", "add-to-db", "some_barcode"]) + result = runner.invoke(app, ["digifeeds", "add-to-digifeeds-set", "some_barcode"]) assert "Item not found in alma" in result.stdout assert "Item NOT added to digifeeds set" in result.stdout diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_digifeeds_set.py similarity index 82% rename from tests/digifeeds/test_add_to_db.py rename to tests/digifeeds/test_add_to_digifeeds_set.py index 669f99e..5d8177c 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_digifeeds_set.py @@ -3,7 +3,7 @@ import json import pytest import responses -from aim.digifeeds.functions import add_to_db +from aim.digifeeds.functions import add_to_digifeeds_set from aim.digifeeds.alma_client import AlmaClient from aim.digifeeds.db_client import DBClient from requests.exceptions import HTTPError @@ -17,11 +17,11 @@ def item_data(): return output -def test_add_to_db_barcode_thats_in_the_digifeeds_set(mocker, item_data): +def test_add_to_digifeeds_set_for_barcode_thats_in_the_digifeeds_set(mocker, item_data): get_item_mock = mocker.patch.object( DBClient, "get_or_add_item", return_value=item_data ) - result = add_to_db("my_barcode") + result = add_to_digifeeds_set("my_barcode") get_item_mock.assert_called_once() assert result.barcode == "some_barcode" @@ -50,22 +50,24 @@ def test_add_to_db_barcode_thats_in_the_digifeeds_set_but_doesnt_have_status( }, } add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" - add_to_digifeeds_set = responses.post( + add_to_digifeeds_set_stub = responses.post( add_to_digifeeds_url, json=error_body, status=400, ) - result = add_to_db("some_barcode") + result = add_to_digifeeds_set("some_barcode") get_item_mock.assert_called_once() add_status_mock.assert_called_once_with( barcode="some_barcode", status="added_to_digifeeds_set" ) - assert add_to_digifeeds_set.call_count == 1 + assert add_to_digifeeds_set_stub.call_count == 1 assert result.barcode == "some_barcode" -def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): +def test_add_to_digifeeds_set_for_barcode_thats_not_in_the_digifeeds_set( + mocker, item_data +): item_data["statuses"][0]["name"] = "some_other_status" get_item_mock = mocker.patch.object( DBClient, "get_or_add_item", return_value=item_data @@ -78,7 +80,7 @@ def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): "add_barcode_to_digifeeds_set", return_value=item_data, ) - result = add_to_db("some_barcode") + result = add_to_digifeeds_set("some_barcode") get_item_mock.assert_called_once() add_status_mock.assert_called_once() add_to_digifeeds_set_mock.assert_called_once() @@ -86,7 +88,7 @@ def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): @responses.activate -def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): +def test_add_to_digifeeds_set_barcode_that_is_not_in_alma(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" get_item_mock = mocker.patch.object( DBClient, "get_or_add_item", return_value=item_data @@ -107,23 +109,23 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): }, } add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" - add_to_digifeeds_set = responses.post( + add_to_digifeeds_set_stub = responses.post( add_to_digifeeds_url, json=error_body, status=400, ) - result = add_to_db("some_barcode") + result = add_to_digifeeds_set("some_barcode") get_item_mock.assert_called_once() add_status_mock.assert_called_once_with( barcode="some_barcode", status="not_found_in_alma" ) - assert add_to_digifeeds_set.call_count == 1 + assert add_to_digifeeds_set_stub.call_count == 1 assert result.barcode == "some_barcode" @responses.activate -def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data): +def test_add_to_digifeeds_set_barcode_that_causes_alma_error(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" get_item_mock = mocker.patch.object( DBClient, "get_or_add_item", return_value=item_data @@ -142,15 +144,15 @@ def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data): }, } add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" - add_to_digifeeds_set = responses.post( + add_to_digifeeds_set_stub = responses.post( add_to_digifeeds_url, json=error_body, status=400, ) with pytest.raises(Exception) as exc_info: - add_to_db("my_barcode") + add_to_digifeeds_set("my_barcode") assert exc_info.type is HTTPError - assert add_to_digifeeds_set.call_count == 1 + assert add_to_digifeeds_set_stub.call_count == 1 get_item_mock.assert_called_once() add_status_mock.assert_not_called() From 66b53ae0c1f51b939c1103faf6cf26c983c731b6 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 13 Nov 2024 20:54:06 +0000 Subject: [PATCH 6/6] every step of processing barcodes can add a barcode to the db --- aim/digifeeds/functions.py | 13 ++----------- tests/digifeeds/test_check_zephir.py | 13 +++---------- tests/digifeeds/test_move_to_pickup.py | 13 ++----------- 3 files changed, 7 insertions(+), 32 deletions(-) diff --git a/aim/digifeeds/functions.py b/aim/digifeeds/functions.py index d206b2f..87b9b44 100644 --- a/aim/digifeeds/functions.py +++ b/aim/digifeeds/functions.py @@ -54,11 +54,7 @@ def add_to_digifeeds_set(barcode: str): def move_to_pickup(barcode: str): - raw_item = DBClient().get_item(barcode) - if raw_item is None: - raise Exception("Item not found in database") - - item = Item(raw_item) + item = Item(DBClient().get_or_add_item(barcode)) if not item.in_zephir_for_long_enough: return None @@ -94,12 +90,7 @@ def list_barcodes_in_input_bucket(): def check_zephir(barcode: str): - raw_item = DBClient().get_item(barcode) - if raw_item is None: - raise Exception("Item not found in database") - - item = Item(raw_item) - + item = Item(DBClient().get_or_add_item(barcode)) if item.has_status("in_zephir"): return item diff --git a/tests/digifeeds/test_check_zephir.py b/tests/digifeeds/test_check_zephir.py index 55d98bc..3afcadb 100644 --- a/tests/digifeeds/test_check_zephir.py +++ b/tests/digifeeds/test_check_zephir.py @@ -21,7 +21,7 @@ def barcode(): @responses.activate def test_barcode_is_in_zephir(mocker, item_data, barcode): get_item_mock = mocker.patch.object( - functions.DBClient, "get_item", return_value=item_data + functions.DBClient, "get_or_add_item", return_value=item_data ) add_status_mock = mocker.patch.object( functions.DBClient, "add_item_status", return_value=item_data @@ -37,7 +37,7 @@ def test_barcode_is_in_zephir(mocker, item_data, barcode): def test_barcode_already_has_in_zephir_status(mocker, item_data, barcode): item_data["statuses"][0]["name"] = "in_zephir" get_item_mock = mocker.patch.object( - functions.DBClient, "get_item", return_value=item_data + functions.DBClient, "get_or_add_item", return_value=item_data ) add_status_mock = mocker.patch.object( @@ -53,7 +53,7 @@ def test_barcode_already_has_in_zephir_status(mocker, item_data, barcode): @responses.activate def test_barcode_not_in_zephir(mocker, item_data, barcode): get_item_mock = mocker.patch.object( - functions.DBClient, "get_item", return_value=item_data + functions.DBClient, "get_or_add_item", return_value=item_data ) add_status_mock = mocker.patch.object( @@ -65,10 +65,3 @@ def test_barcode_not_in_zephir(mocker, item_data, barcode): get_item_mock.assert_called_once() add_status_mock.assert_not_called() assert result is None - - -def test_barcdoe_is_not_in_db(mocker, barcode): - mocker.patch.object(functions.DBClient, "get_item", return_value=None) - with pytest.raises(Exception) as exc_info: - check_zephir(barcode) - assert exc_info.type is Exception diff --git a/tests/digifeeds/test_move_to_pickup.py b/tests/digifeeds/test_move_to_pickup.py index a7b2ad9..5cefd44 100644 --- a/tests/digifeeds/test_move_to_pickup.py +++ b/tests/digifeeds/test_move_to_pickup.py @@ -30,7 +30,7 @@ def test_move_to_pickup_success(mocker, item_in_zephir_for_long_enough): rclone_moveto_mock = mocker.patch.object(rclone, "moveto") get_item_mock = mocker.patch.object( DBClient, - "get_item", + "get_or_add_item", return_value=item_in_zephir_for_long_enough, ) add_status_mock = mocker.patch.object( @@ -48,19 +48,10 @@ def test_move_to_pickup_success(mocker, item_in_zephir_for_long_enough): assert result is not None -def test_move_to_pickup_no_item(mocker): - get_item_mock = mocker.patch.object(DBClient, "get_item", return_value=None) - with pytest.raises(Exception) as exc_info: - move_to_pickup("some_barcode") - - get_item_mock.assert_called_once() - assert str(exc_info.value) == "Item not found in database" - - def test_move_to_pickup_item_too_recent(mocker, item_in_zephir_too_recent): get_item_mock = mocker.patch.object( DBClient, - "get_item", + "get_or_add_item", return_value=item_in_zephir_too_recent, ) result = move_to_pickup(item_in_zephir_too_recent["barcode"])