Skip to content

Skip manual AutoTEM #833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tieneupin merged 2 commits into from
Jun 23, 2026
Merged

Skip manual AutoTEM #833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1aa4c73
Add logic to distinguish between manual and automated AutoTEM project…
tieneupin Jun 22, 2026
0d624a9
Add test to check for manual AutoTEM workflow
tieneupin Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 110 additions & 83 deletions src/murfey/client/contexts/fib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,23 @@ def _parse_boolean(text: str):
)


def _is_manual_autotem(file_path: Path):
"""
Checks if the file being analysed belongs to a manual AutoTEM project.
Manual AutoTEM projects are stored in project folders that start with 'AutoTEM_'.
"""
try:
autotem_idx = file_path.parts.index("autotem")
project_dir = file_path.parents[-(autotem_idx + 2)]
return project_dir.stem.startswith("AutoTEM_")
except Exception:
logger.error(
f"Error checking if file {file_path} belongs to a manual AutoTEM project:",
exc_info=True,
)
return None


def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None:
"""
Returns the Path of the file on the client PC.
Expand Down Expand Up @@ -246,94 +263,104 @@ def post_transfer(
# AutoTEM
# -----------------------------------------------------------------------------
if self._acquisition_software == "autotem":
if transferred_file.name == "ProjectData.dat":
logger.info(f"Found metadata file {transferred_file} for parsing")

# Parse the metadata file
all_site_info_new = self._parse_autotem_metadata(transferred_file)
for site_num, site_info_new in all_site_info_new.items():
# Post the data to the backend if it's been changed
if (
data := site_info_new.model_dump(exclude_none=True)
) != self._site_info.get(site_num, LamellaSiteInfo()).model_dump(
exclude_none=True
):
capture_post(
base_url=str(environment.url.geturl()),
router_name="workflow_fib.router",
function_name="register_fib_milling_progress",
token=self._token,
instrument_name=environment.instrument_name,
data=data,
# Endpoint kwargs
session_id=environment.murfey_session,
)
# Apply different logic depending of whether it's auto/manual AutoTEM
if (is_manual_autotem := _is_manual_autotem(transferred_file)) is None:
# Skip processing file if the check fails
return None
elif is_manual_autotem:
# Logic for handling manual AutotTEM will evenutlaly go here
return None
else:
# Extract metadata from fully automated AutoTEM projects
if transferred_file.name == "ProjectData.dat":
logger.info(f"Found metadata file {transferred_file} for parsing")

# Parse the metadata file
all_site_info_new = self._parse_autotem_metadata(transferred_file)
for site_num, site_info_new in all_site_info_new.items():
# Post the data to the backend if it's been changed
if (
data := site_info_new.model_dump(exclude_none=True)
) != self._site_info.get(
site_num, LamellaSiteInfo()
).model_dump(exclude_none=True):
capture_post(
base_url=str(environment.url.geturl()),
router_name="workflow_fib.router",
function_name="register_fib_milling_progress",
token=self._token,
instrument_name=environment.instrument_name,
data=data,
# Endpoint kwargs
session_id=environment.murfey_session,
)

# Update existing dict
self._site_info[site_num] = site_info_new
logger.info(f"Updating metadata for site {site_num}")

# Post drift correction GIF request if it hasn't already been done
fib_image = self._drift_correction_images.get(site_num, None)
if fib_image is not None and not fib_image.is_submitted:
# Construct the output file name if it doesn't already exist
if (output_file := fib_image.output_file) is None:
source = _get_source(transferred_file, environment)
if source is None:
logger.warning(
f"No source found for file {transferred_file}"
# Update existing dict
self._site_info[site_num] = site_info_new
logger.info(f"Updating metadata for site {site_num}")

# Post drift correction GIF request if it hasn't already been done
fib_image = self._drift_correction_images.get(site_num, None)
if fib_image is not None and not fib_image.is_submitted:
# Construct the output file name if it doesn't already exist
if (output_file := fib_image.output_file) is None:
source = _get_source(transferred_file, environment)
if source is None:
logger.warning(
f"No source found for file {transferred_file}"
)
continue
destination_file = _file_transferred_to(
environment=environment,
source=source,
file_path=transferred_file,
rsync_basepath=Path(
self._machine_config.get("rsync_basepath", "")
),
)
continue
destination_file = _file_transferred_to(
environment=environment,
source=source,
file_path=transferred_file,
rsync_basepath=Path(
self._machine_config.get("rsync_basepath", "")
),
)
if destination_file is None:
logger.warning(
f"Could not find destination file path for {transferred_file.name!r}"
if destination_file is None:
logger.warning(
f"Could not find destination file path for {transferred_file.name!r}"
)
continue
output_dir = self._determine_output_dir(
site_num, destination_file, environment
)
continue
output_dir = self._determine_output_dir(
site_num, destination_file, environment
)
if output_dir is None:
logger.warning(
f"Could not determine output directory for lamella {site_num}"
if output_dir is None:
logger.warning(
f"Could not determine output directory for lamella {site_num}"
)
continue
output_file = (
output_dir
/ "drift_correction"
/ f"lamella_{site_num}.gif"
)
continue
output_file = (
output_dir
/ "drift_correction"
/ f"lamella_{site_num}.gif"
)
with lock:
self._drift_correction_images[
site_num
].output_file = output_file
# Reload the new object
fib_image = self._drift_correction_images[site_num]

if self._make_gif(
environment=environment,
lamella_number=site_num,
images=sorted(fib_image.images),
output_file=output_file,
):
with lock:
self._drift_correction_images[
site_num
].is_submitted = True
return None
with lock:
self._drift_correction_images[
site_num
].output_file = output_file
# Reload the new object
fib_image = self._drift_correction_images[site_num]

if self._make_gif(
environment=environment,
lamella_number=site_num,
images=sorted(fib_image.images),
output_file=output_file,
):
with lock:
self._drift_correction_images[
site_num
].is_submitted = True
return None

elif (
"DCImages" in transferred_file.parts
and transferred_file.suffix == ".png"
):
self._make_drift_correction_gif(transferred_file, environment)
elif (
"DCImages" in transferred_file.parts
and transferred_file.suffix == ".png"
):
self._make_drift_correction_gif(transferred_file, environment)
return None

# -----------------------------------------------------------------------------
# Maps
Expand Down
57 changes: 52 additions & 5 deletions tests/client/contexts/test_fib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ def _create_activity_node(


def _create_site_node(
site_prefix: str,
site_num: int,
has_site_name: bool = True,
has_recipes: bool = True,
Expand All @@ -233,9 +234,14 @@ def _create_site_node(

if has_site_name:
name_node = ET.Element("Name")
name_node.text = "Lamella"
if site_num > 1:
name_node.text += f" ({site_num})"
name_node.text = site_prefix
# If the site name starts with "Lamella"
if site_prefix == "Lamella":
if site_num > 1:
name_node.text += f" ({site_num})"
# If the site name starts with "Site"
else:
name_node.text += f" #{site_num}"
site_node.append(name_node)

# Create the stage position nodes
Expand Down Expand Up @@ -296,6 +302,8 @@ def _create_site_node(

def create_fib_autotem_project_data(
visit_dir: Path,
project_name: str,
site_prefix: str,
has_project_name: bool = True,
has_sites: bool = True,
has_site_name: bool = True,
Expand All @@ -320,6 +328,7 @@ def create_fib_autotem_project_data(
n += 1
site_parent_node.append(
_create_site_node(
site_prefix,
n,
has_site_name=has_site_name,
has_recipes=has_recipes,
Expand Down Expand Up @@ -466,7 +475,7 @@ def test_file_transferred_to(
(False, True, True, True, True, True, True, True, False), # No environment
),
)
def test_fib_autotem_context_projectdata(
def test_fib_full_autotem_context_projectdata(
mocker: MockerFixture,
test_params: tuple[bool, bool, bool, bool, bool, bool, bool, bool, bool],
tmp_path: Path,
Expand Down Expand Up @@ -530,6 +539,8 @@ def test_fib_autotem_context_projectdata(
# Create the mock metadata file to parse
mock_projectdata = create_fib_autotem_project_data(
visit_dir=visit_dir,
project_name=project_name,
site_prefix="Lamella",
has_project_name=has_project_name,
has_sites=has_sites,
has_site_name=has_site_name,
Expand Down Expand Up @@ -677,7 +688,7 @@ def test_fib_autotem_context_projectdata(
(True, True, True, True, True, True, True),
),
)
def test_fib_autotem_context_drift_correction_images(
def test_fib_full_autotem_context_drift_correction_images(
mocker: MockerFixture,
test_params: tuple[bool, bool, bool, bool, bool, bool, bool],
tmp_path: Path,
Expand Down Expand Up @@ -815,6 +826,42 @@ def test_fib_autotem_context_drift_correction_images(
assert mock_capture_post.call_count == len(destination_files)


def test_fib_manual_autotem_context_projectdata(
mocker: MockerFixture,
visit_dir: Path,
):
# Mock the ProjectData.dat file
mock_projectdata = create_fib_autotem_project_data(
visit_dir=visit_dir,
project_name=f"AutoTEM_200101-1200_{project_name}",
site_prefix="Site",
)

# Mock the Murfey environment
mock_environment = MagicMock()
mock_environment.visit = visit_name

# Patch the '_parse_autotem_metadata' class function
mock_parse = mocker.patch.object(FIBContext, "_parse_autotem_metadata")

# Mock the functions used in 'post_transfer'
mock_capture_post = mocker.patch("murfey.client.contexts.fib.capture_post")

# Initialise the FIBContext
basepath = visit_dir
context = FIBContext(
acquisition_software="autotem",
basepath=basepath,
machine_config={},
token="",
)

# Pass file to FIBContext and check that it behaves as expected
context.post_transfer(mock_projectdata, environment=mock_environment)
mock_parse.assert_not_called()
mock_capture_post.assert_not_called()


def test_fib_maps_context(
mocker: MockerFixture,
tmp_path: Path,
Expand Down