Selective upstream file download

src/murfey/instrument_server/api.py

@@ -476,17 +476,22 @@ class UpstreamFileDownloadInfo(BaseModel):
     download_dir: Path
     upstream_instrument: str
     upstream_visit_path: Path
+    search_strings: list[str] | None = None
 
 
 @router.post("/visits/{visit_name}/sessions/{session_id}/upstream_file_data_request")
-def gather_upstream_files(
+def run_upstream_file_download_request(
     visit_name: str,
     session_id: MurfeySessionID,
     upstream_file_download: UpstreamFileDownloadInfo,
 ):
     """
-    Instrument server endpoint that will query the backend for files in the chosen
-    visit directory
+    Instrument server endpoint that will receive an order from the backend server
+    to trigger an upstream file download request.
+
+    It will get the list of files matching the provided list of search strings,
+    then iteratively request for those files, saving them locally at the specified
+    download directory.
     """
     # Check for forbidden characters
     if any(c in visit_name for c in ("/", "\\", ":", ";")):
@@ -518,6 +523,7 @@ def gather_upstream_files(
         json={
             "upstream_instrument": upstream_instrument,
             "upstream_visit_path": str(upstream_visit_path),
+            "search_strings": upstream_file_download.search_strings,
         },
     ).json()
 

src/murfey/server/api/instrument.py

@@ -437,7 +437,7 @@ async def request_upstream_tiff_data_download(
 
 
 @router.post("/visits/{visit_name}/sessions/{session_id}/upstream_file_data_request")
-async def request_upstream_file_data_download(
+async def request_upstream_file_download(
     visit_name: str,
     session_id: MurfeySessionID,
     upstream_file_request: UpstreamFileRequestInfo,
@@ -475,7 +475,7 @@ async def request_upstream_file_data_download(
     async with aiohttp.ClientSession() as clientsession:
         url_path = url_path_for(
             "api.router",
-            "gather_upstream_files",
+            "run_upstream_file_download_request",
             visit_name=secure_filename(visit_name),
             session_id=session_id,
         )
@@ -488,6 +488,7 @@ async def request_upstream_file_data_download(
                 "download_dir": download_dir,
                 "upstream_instrument": upstream_file_request.upstream_instrument,
                 "upstream_visit_path": str(upstream_file_request.upstream_visit_path),
+                "search_strings": upstream_file_request.search_strings,
             },
         ) as resp:
             data = await resp.json()

src/murfey/server/api/session_control.py

@@ -554,6 +554,7 @@ async def gather_upstream_files(
         session_id=session_id,
         upstream_instrument=upstream_file_request.upstream_instrument,
         upstream_visit_path=upstream_file_request.upstream_visit_path,
+        search_strings=upstream_file_request.search_strings,
         db=db,
     )
 

src/murfey/server/api/session_info.py

@@ -530,6 +530,7 @@ async def gather_upstream_files(
         session_id=session_id,
         upstream_instrument=upstream_file_request.upstream_instrument,
         upstream_visit_path=upstream_file_request.upstream_visit_path,
+        search_strings=upstream_file_request.search_strings,
         db=db,
     )
 

src/murfey/server/api/session_shared.py

@@ -20,7 +20,7 @@
     Session as MurfeySession,
 )
 
-logger = logging.getLogger("murfey.server.api.shared")
+logger = logging.getLogger("murfey.server.api.session_shared")
 
 
 def remove_session_by_id(session_id: int, db):
@@ -211,44 +211,52 @@ def gather_upstream_files(
     session_id: int,
     upstream_instrument: str,
     upstream_visit_path: Path,
+    search_strings: list[str] | None,
     db: SQLModelSession,
 ):
     """
     Searches the specified upstream instrument for files based on the search strings
     set in the MachineConfig and returns them as a list of file paths.
     """
-    # Load the current instrument's machine config
-    murfey_session = db.exec(
-        select(MurfeySession).where(MurfeySession.id == session_id)
-    ).one()
-    instrument_name = murfey_session.instrument_name
-    machine_config = get_machine_config(instrument_name=instrument_name)[
-        instrument_name
-    ]
-
-    # Search for files using the configured strings for that upstream instrument
     file_list: list[Path] = []
     logger.info(f"Searching for files in {sanitise(str(upstream_visit_path))!r}")
-    if (
-        machine_config.upstream_data_search_strings.get(upstream_instrument, None)
-        is not None
-    ):
-        for search_string in machine_config.upstream_data_search_strings[
-            upstream_instrument
-        ]:
-            logger.info(f"Using search string {search_string}")
-            for file in upstream_visit_path.glob(search_string):
-                if file.is_file():
-                    file_list.append(file)
-        logger.info(
-            f"Found {len(file_list)} files for download "
-            f"from {sanitise(upstream_instrument)}"
+
+    # If search strings weren't provided, read them from the machine config
+    if search_strings is None:
+        # Load the current instrument's machine config
+        murfey_session = db.exec(
+            select(MurfeySession).where(MurfeySession.id == session_id)
+        ).one()
+        instrument_name = murfey_session.instrument_name
+        machine_config = get_machine_config(instrument_name=instrument_name)[
+            instrument_name
+        ]
+        search_strings = machine_config.upstream_data_search_strings.get(
+            upstream_instrument, None
         )
-    else:
+        # Return empty list if no search strings for the instrument were found
+        if search_strings is None:
+            logger.warning(
+                "Upstream file searching has not been configured for "
+                f"{sanitise(upstream_instrument)} on {sanitise(instrument_name)}"
+            )
+            return file_list
+    elif not search_strings:
+        # Return empty list if no search strings were provided to begin with
         logger.warning(
-            "Upstream file searching has not been configured for "
-            f"{sanitise(upstream_instrument)} on {sanitise(instrument_name)}"
+            "No search strings were included as part of the file download request"
         )
+        return file_list
+    # Search for files matching the provided search strings
+    for search_string in search_strings:
+        logger.info(f"Using search string {sanitise(search_string)}")
+        for file in upstream_visit_path.glob(search_string):
+            if file.is_file():
+                file_list.append(file)
+    logger.info(
+        f"Found {len(file_list)} files for download "
+        f"from {sanitise(upstream_instrument)}"
+    )
     return file_list
 
 

src/murfey/util/models.py

@@ -89,6 +89,7 @@ class UpstreamFileRequestInfo(BaseModel):
     # Used in backend server for cross-instrument file download requests
     upstream_instrument: str
     upstream_visit_path: Path
+    search_strings: list[str] | None = None
 
 
 """

src/murfey/util/route_manifest.yaml

@@ -149,7 +149,7 @@ murfey.instrument_server.api.router:
     methods:
       - POST
   - path: /visits/{visit_name}/sessions/{session_id}/upstream_file_data_request
-    function: gather_upstream_files
+    function: run_upstream_file_download_request
     path_params:
       - name: visit_name
         type: str
@@ -569,7 +569,7 @@ murfey.server.api.instrument.router:
     methods:
       - POST
   - path: /instrument_server/visits/{visit_name}/sessions/{session_id}/upstream_file_data_request
-    function: request_upstream_file_data_download
+    function: request_upstream_file_download
     path_params:
       - name: visit_name
         type: str

tests/server/api/test_session_control.py

@@ -1,7 +1,9 @@
 from pathlib import Path
+from typing import Any
 from unittest import mock
 from unittest.mock import MagicMock
 
+import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 from pytest_mock import MockerFixture
@@ -10,9 +12,10 @@
     validate_instrument_server_session_access,
     validate_instrument_token,
 )
-from murfey.server.api.session_control import spa_router
+from murfey.server.api.session_control import gather_upstream_files, spa_router
 from murfey.server.murfey_db import murfey_db_session
 from murfey.util.api import url_path_for
+from murfey.util.models import UpstreamFileRequestInfo
 
 
 def test_make_atlas_jpg(mocker: MockerFixture, tmp_path: Path):
@@ -74,3 +77,55 @@ def mock_get_db_session():
     # Check that the expected calls were made
     mock_atlas_jpg.assert_called_once_with(instrument_name, visit_name, test_file)
     assert response.status_code == 200
+
+
+@pytest.mark.parametrize(
+    "search_strings",
+    (
+        ["dummy"],
+        [],
+        None,
+    ),
+)
+@pytest.mark.asyncio
+async def test_gather_upstream_files(
+    mocker: MockerFixture,
+    tmp_path: Path,
+    search_strings: list[str] | None,
+):
+    # Construct dictionary to pass to Pydantic model
+    session_id = 1
+    upstream_instrument = "dummy"
+    upstream_visit_path = str(tmp_path / "dummy")
+    params_dict: dict[str, Any] = {
+        "upstream_instrument": upstream_instrument,
+        "upstream_visit_path": upstream_visit_path,
+    }
+    if search_strings is not None:
+        params_dict["search_strings"] = search_strings
+
+    # Validate the incoming message
+    params = UpstreamFileRequestInfo(**params_dict)
+
+    # Patch the actual 'gather_upstream_files' function
+    mock_gather = mocker.patch(
+        "murfey.server.api.session_control._gather_upstream_files"
+    )
+
+    # Create a mock database session
+    mock_db = MagicMock()
+
+    # Run the function and check that the expected calls were made:
+    await gather_upstream_files(
+        visit_name="dummy",
+        session_id=session_id,
+        upstream_file_request=params,
+        db=mock_db,
+    )
+    mock_gather.assert_called_with(
+        session_id=session_id,
+        upstream_instrument=upstream_instrument,
+        upstream_visit_path=Path(upstream_visit_path),
+        search_strings=search_strings,
+        db=mock_db,
+    )

tests/server/api/test_session_info.py

@@ -0,0 +1,59 @@
+from pathlib import Path
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+from pytest_mock import MockerFixture
+
+from murfey.server.api.session_info import gather_upstream_files
+from murfey.util.models import UpstreamFileRequestInfo
+
+
+@pytest.mark.parametrize(
+    "search_strings",
+    (
+        ["dummy"],
+        [],
+        None,
+    ),
+)
+@pytest.mark.asyncio
+async def test_gather_upstream_files(
+    mocker: MockerFixture,
+    tmp_path: Path,
+    search_strings: list[str] | None,
+):
+    # Construct dictionary to pass to Pydantic model
+    session_id = 1
+    upstream_instrument = "dummy"
+    upstream_visit_path = str(tmp_path / "dummy")
+    params_dict: dict[str, Any] = {
+        "upstream_instrument": upstream_instrument,
+        "upstream_visit_path": upstream_visit_path,
+    }
+    if search_strings is not None:
+        params_dict["search_strings"] = search_strings
+
+    # Validate the incoming message
+    params = UpstreamFileRequestInfo(**params_dict)
+
+    # Patch the actual 'gather_upstream_files' function
+    mock_gather = mocker.patch("murfey.server.api.session_info._gather_upstream_files")
+
+    # Create a mock database session
+    mock_db = MagicMock()
+
+    # Run the function and check that the expected calls were made:
+    await gather_upstream_files(
+        visit_name="dummy",
+        session_id=session_id,
+        upstream_file_request=params,
+        db=mock_db,
+    )
+    mock_gather.assert_called_with(
+        session_id=session_id,
+        upstream_instrument=upstream_instrument,
+        upstream_visit_path=Path(upstream_visit_path),
+        search_strings=search_strings,
+        db=mock_db,
+    )