Merge pull request #19 from mlibrary/move-digifeeds-functions-to-module

Move digifeeds functions to module

aim/cli/digifeeds.py

@@ -4,11 +4,9 @@
 
 import typer
 from typing_extensions import Annotated
-from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db
-from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket
-from aim.digifeeds.check_zephir import check_zephir as check_zephir_for_barcode
-from aim.digifeeds.move_to_pickup import move_to_pickup as move_volume_to_pickup
 from aim.digifeeds.database import models, main
+from aim.digifeeds import functions
+
 import json
 import sys
 
@@ -17,7 +15,7 @@
 
 
 @app.command()
-def add_to_db(
+def add_to_digifeeds_set(
     barcode: Annotated[
         str,
         typer.Argument(help="The barcode to be added to the database"),
@@ -34,7 +32,7 @@ def add_to_db(
         barcode (str): Barcode of item
     """
     print(f'Adding barcode "{barcode}" to database')
-    item = add_to_digifeeds_db(barcode)
+    item = functions.add_to_digifeeds_set(barcode)
     if item.has_status("not_found_in_alma"):
         print("Item not found in alma.")
     if item.has_status("added_to_digifeeds_set"):
@@ -57,7 +55,7 @@ def check_zephir(
     """
 
     print(f"Checking Zephir for {barcode}")
-    item = check_zephir_for_barcode(barcode)
+    item = functions.check_zephir(barcode)
     if item:
         print(f"{barcode} is in Zephir")
     else:
@@ -78,7 +76,7 @@ def list_barcodes_in_input_bucket():
     """
     List the barcodes currently in the input directory in the S3 bucket.
     """
-    json.dump(list_barcodes_in_bucket(), sys.stdout)
+    json.dump(functions.list_barcodes_in_input_bucket(), sys.stdout)
 
 
 @app.command()
@@ -94,7 +92,7 @@ def move_to_pickup(
     folder in the bucket and prefixed with the date and time.
     """
     print(f'Moving barcode "{barcode}" from the s3 bucket to the google drive')
-    item = move_volume_to_pickup(barcode)
+    item = functions.move_to_pickup(barcode)
     if item is None:
         print("Item has not been in zephir long enough")
     else:

aim/digifeeds/functions.py

@@ -0,0 +1,102 @@
+from aim.services import S
+from aim.digifeeds.db_client import DBClient
+from aim.digifeeds.item import Item
+import requests
+import boto3
+from pathlib import Path
+from rclone_python import rclone
+from datetime import datetime
+from aim.digifeeds.alma_client import AlmaClient
+from requests.exceptions import HTTPError
+
+
+def add_to_digifeeds_set(barcode: str):
+    """
+    Adds a barcode to the digifeeds set and to the digifeeds database if it
+    doesn't already exist. If the barcode is in the digifeeds set already but
+    doesn't have an added_to_digifeeds_set status in the digifeeds db, that
+    status gets added.
+
+    Args:
+        barcode (str): Barcode of the item
+
+    Raises:
+        ext_inst: HTTPError
+
+    Returns:
+        aim.digifeeds.database.models.Item: Item object
+    """
+    item = Item(DBClient().get_or_add_item(barcode))
+    if not item.has_status("added_to_digifeeds_set"):
+        try:
+            AlmaClient().add_barcode_to_digifeeds_set(barcode)
+        except HTTPError as ext_inst:
+            errorList = ext_inst.response.json()["errorList"]["error"]
+            if any(e["errorCode"] == "60120" for e in errorList):
+                if not item.has_status("not_found_in_alma"):
+                    item = Item(
+                        DBClient().add_item_status(
+                            barcode=barcode, status="not_found_in_alma"
+                        )
+                    )
+                return item
+            elif any(e["errorCode"] == "60115" for e in errorList):
+                # 60115 means the barcode is already in the set. That means the
+                # db entry from this barcdoe needs to have
+                # added_to_digifeeds_set
+                pass
+            else:
+                raise ext_inst
+        item = Item(
+            DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set")
+        )
+    return item
+
+
+def move_to_pickup(barcode: str):
+    item = Item(DBClient().get_or_add_item(barcode))
+
+    if not item.in_zephir_for_long_enough:
+        return None
+
+    DBClient().add_item_status(barcode=barcode, status="copying_start")
+    rclone.copyto(
+        f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}/{barcode}.zip",
+        f"{S.digifeeds_gdrive_rclone_remote}:{barcode}.zip",
+    )
+    DBClient().add_item_status(barcode=barcode, status="copying_end")
+    timestamp = datetime.now().strftime("%F_%H-%M-%S")
+    rclone.moveto(
+        f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}/{barcode}.zip",
+        f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_processed_path}/{timestamp}_{barcode}.zip",
+    )
+    final_raw_item = DBClient().add_item_status(
+        barcode=barcode, status="pending_deletion"
+    )
+
+    return final_raw_item
+
+
+def list_barcodes_in_input_bucket():
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=S.digifeeds_s3_access_key,
+        aws_secret_access_key=S.digifeeds_s3_secret_access_key,
+    )
+    prefix = S.digifeeds_s3_input_path + "/"
+    response = s3.list_objects_v2(Bucket=S.digifeeds_s3_bucket, Prefix=prefix)
+    barcodes = [Path(object["Key"]).stem for object in response["Contents"]]
+    return barcodes
+
+
+def check_zephir(barcode: str):
+    item = Item(DBClient().get_or_add_item(barcode))
+    if item.has_status("in_zephir"):
+        return item
+
+    response = requests.get(f"{S.zephir_bib_api_url}/mdp.{barcode}")
+    if response.status_code == 200:
+        DBClient().add_item_status(barcode=barcode, status="in_zephir")
+        return item
+    else:
+        return None

docs/api/aim.digifeeds.functions.rst

@@ -0,0 +1,7 @@
+aim.digifeeds.functions module
+==============================
+
+.. automodule:: aim.digifeeds.functions
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/api/aim.digifeeds.rst

@@ -20,10 +20,7 @@ Submodules
 .. toctree::
    :maxdepth: 4
 
-   aim.digifeeds.add_to_db
    aim.digifeeds.alma_client
-   aim.digifeeds.check_zephir
    aim.digifeeds.db_client
+   aim.digifeeds.functions
    aim.digifeeds.item
-   aim.digifeeds.list_barcodes_in_bucket
-   aim.digifeeds.move_to_pickup