From 4cf5aa2ed05beb6df93f415d69b21905840837ca Mon Sep 17 00:00:00 2001 From: Victor Perez Meza Date: Wed, 20 Mar 2024 12:35:59 +0100 Subject: [PATCH 01/11] catch empty cycles --- macsima2mc.py | 59 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/macsima2mc.py b/macsima2mc.py index 4743ed5..98ecd31 100644 --- a/macsima2mc.py +++ b/macsima2mc.py @@ -1,4 +1,5 @@ #!/usr/bin/python +import json import ome_types import numpy as np import pandas as pd @@ -61,7 +62,7 @@ parser.add_argument('-he', '--hi_exposure_only', action='store_true', - help='Activate this flag to extract only the set of images with the highest exposure time.' + help='Activate this flag to extract only the set o-of images with the highest exposure time.' ) parser.add_argument('-nb', @@ -109,7 +110,13 @@ def antigen_cycle_info(antigen_cycle_no=antigen_cycle_number,cycles_path=cycles_ antigen_cycle=f'{antigen_cycle_no:03d}' cycle_folder='_'.join([antigen_cycle,'AntigenCycle']) workdir=os.path.join(cycles_path,cycle_folder) + #filter out non tif files images=list(filter(lambda x: x.endswith('.tif'),os.listdir(workdir))) + #filter out files with tags None or Empty in the name + #images=list(filter(lambda x: "Empty" not in x,images)) + #filter out files with tags None or Empty in the name + #images=list(filter(lambda x: "None" not in x,images)) + cycle_info={'img_full_path':[], 'image':images, 'marker':[], @@ -121,6 +128,8 @@ def antigen_cycle_info(antigen_cycle_no=antigen_cycle_number,cycles_path=cycles_ 'exposure':[] } + + for im in images: marker_info=im.split('AntigenCycle')[-1].split('__') @@ -133,6 +142,11 @@ def antigen_cycle_info(antigen_cycle_no=antigen_cycle_number,cycles_path=cycles_ if ref_marker in m: cycle_info['marker'].append(ref_marker) cycle_info['filter'].append(ref_marker) + + elif "Empty" in m: + cycle_info['marker'].append(m.split('_')[0]) + cycle_info['filter'].append(m.split("bit")[0].split("_")[-2]) + else: cycle_info['marker'].append(m) #cycle_info['filter'].append(marker_info[-1].split('_')[2]) @@ -183,6 +197,9 @@ def bleach_cycle_info(antigen_cycle_no=antigen_cycle_number,cycles_path=cycles_d cycle_folder='_'.join([bleach_cycle,'BleachCycle']) workdir=os.path.join(cycles_path,cycle_folder) images=list(filter(lambda x: x.endswith('.tif'),os.listdir(workdir))) + #filter out files with tags None or Empty in the name + #images=list(filter(lambda x: "Empty" not in x,images)) + #images=list(filter(lambda x: "None" not in x,images)) cycle_info={'img_full_path':[], 'image':images, 'marker':[], @@ -297,12 +314,13 @@ def create_ome(img_info,xy_tile_positions_units,img_path): ) #--Generate channels block--# chann_block=[] - for ch in range(0,no_of_channels): + #for ch in range(0,no_of_channels): + for ch,chann_name in enumerate(markers): chann_block.append(Channel(id=ome_types.model.simple_types.ChannelID('Channel:{x}'.format(x=ch)), - color=ome_types.model.simple_types.Color((255,255,255)), - emission_wavelength=1,#place holder - excitation_wavelength=1,#place holder - + name= chann_name, + color=ome_types.model.simple_types.Color((255,255,255)), + emission_wavelength=1,#place holder + excitation_wavelength=1,#place holder ) ) #--Generate pixels block--# @@ -316,7 +334,8 @@ def create_ome(img_info,xy_tile_positions_units,img_path): template_plane_block[ch].position_x=xy_tile_positions_units[t][0] template_plane_block[ch].position_y=xy_tile_positions_units[t][1] template_plane_block[ch].exposure_time=exposures[ch] - template_chann_block[ch].id='Channel:{y}:{x}:{marker_name}'.format(x=ch,y=100+t,marker_name=mark) + template_chann_block[ch].id='Channel:{y}:{x}'.format(x=ch,y=100+t) + template_chann_block[ch].name=mark template_tiffdata_block[ch].ifd=ifd_counter ifd_counter+=1 pix_block.append(Pixels(id=ome_types.model.simple_types.PixelsID('Pixels:{x}'.format(x=t)), @@ -451,7 +470,8 @@ def create_stack(info,exp_level=1, groupA=info.loc[(info['rack']==r) & (info['well']==w)] rois=groupA['roi'].unique() - + print("remember to delete the line below line 467") + rois=[1] for roi in rois: roi_no='roi_{n}'.format(n=f'{roi:02d}') exp_level_no='exp_level_{n}'.format(n=f'{exp_level:02d}') @@ -459,8 +479,11 @@ def create_stack(info,exp_level=1, #create_dir(roi_path) #exposure_path=roi_path / 'exp_level_{n}'.format(n=f'{exp_level:02d}') #create_dir(exposure_path) - output_levels.append(roi_no) - output_levels.append(exp_level_no) + root_name=copy.deepcopy(output_levels) + root_name.append(roi_no) + root_name.append(exp_level_no) + #output_levels.append(roi_no) + #output_levels.append(exp_level_no) counter=0 @@ -484,12 +507,15 @@ def create_stack(info,exp_level=1, X=[] Y=[] + stack=np.zeros((stack_size_z,height,width),dtype=dtype_ref) exposure_per_marker=[] exp=groupB.loc[(groupB['marker']==ref_marker) & (groupB['fov']==1) & (groupB['exposure_level']=='ref'),'exposure'].tolist()[0] exposure_per_marker.append(exp) for s in markers_subset: + #print(s) + #print("test",groupB.loc[(groupB['marker']==s) & (groupB['fov']==1) & (groupB['exposure_level']==exp_level),'exposure'].tolist()) exp=groupB.loc[(groupB['marker']==s) & (groupB['fov']==1) & (groupB['exposure_level']==exp_level),'exposure'].tolist()[0] exposure_per_marker.append(exp) @@ -518,9 +544,9 @@ def create_stack(info,exp_level=1, counter+=1 if args.output_folders_list: - output_folders_path=stack_path / '--'.join(output_levels) / 'raw' + output_folders_path=stack_path / '--'.join(root_name) / 'raw' else: - output_folders_path=stack_path / Path('/'.join(output_levels)) / 'raw' + output_folders_path=stack_path / Path('/'.join(root_name)) / 'raw' if os.path.exists(output_folders_path): @@ -579,6 +605,10 @@ def main(): antigen_info=antigen_cycle_info(antigen_cycle_no=i) exp=antigen_info['exposure_level'].unique() exp=exp[exp!='ref'] + + if len(exp)==0: + exp=[1] + exp.sort() if args.hi_exposure_only: @@ -625,6 +655,11 @@ def main(): else: out_ant['background'].extend(antigen_stack_info['no_channels']*['']) + #with open(Path("D:/out_ant.json"), "w") as fp: + # json.dump(out_ant , fp) + + #with open(Path("D:/out_ble.json"), "w") as fp: + # json.dump(out_ble , fp) for e in exp: if extract_bleach: From 4a750ef2935cacbf949fe7d56174b531e108ece0 Mon Sep 17 00:00:00 2001 From: mbruhns Date: Fri, 22 Mar 2024 15:16:20 +0100 Subject: [PATCH 02/11] Create single function for information extraction. --- nfmacsima.py | 227 ++++++++++++++++++--------------------------------- 1 file changed, 79 insertions(+), 148 deletions(-) diff --git a/nfmacsima.py b/nfmacsima.py index b1cb159..5defb93 100644 --- a/nfmacsima.py +++ b/nfmacsima.py @@ -17,11 +17,12 @@ from ome_types.model import OME, Image, Pixels, TiffData, Channel, Plane from ome_types.model.simple_types import PixelsID, ImageID, Color, ChannelID from ome_types.model.tiff_data import UUID +import re +import sys # ---CLI-BLOCK---# parser = argparse.ArgumentParser() - parser.add_argument( "-i", "--input", @@ -54,7 +55,6 @@ help="Activate this flag to extract only the set of images with the highest exposure time.", ) - args = parser.parse_args() # ---END_CLI-BLOCK---# @@ -75,159 +75,87 @@ # ---- HELPER FUNCTIONS ----# -def antigen_cycle_info(antigen_folder=antigen_dir, ref_marker=ref_marker): - images = [x for x in os.listdir(antigen_folder) if x.endswith(".tif")] +def extract_values(pattern, strings, number_cast=True): + return [ + (int(m.group(1)) if number_cast else m.group(1)) + if (m := re.search(pattern, s)) + else None + for s in strings + ] - cycle_info = { - "img_full_path": [], - "image": images, - "marker": [], - "filter": [], - "rack": [], - "well": [], - "roi": [], - "fov": [], - "exposure": [], - } - for im in images: - # Extraction of marker name info and acquisition info (rack,well,exposure,roi) - marker_info = im.split("AntigenCycle")[-1].split("__") - acq_info = im.split("sensor")[-1].split("_") - # add the information to the cycle_info dictionary - # img full path - cycle_info["img_full_path"].append(os.path.join(antigen_folder, im)) - # marker and fluorophore () - m = marker_info[0].strip("_") - if ref_marker in m: - cycle_info["marker"].append(ref_marker) - cycle_info["filter"].append(ref_marker) - else: - cycle_info["marker"].append(m) - cycle_info["filter"].append(marker_info[-1].split("bit")[0].split("_")[-2]) - - # rack - cycle_info["rack"].append(acq_info[2].split("-")[-1]) - # well - cycle_info["well"].append(acq_info[3].split("-")[-1]) - # roi - cycle_info["roi"].append(acq_info[4].split("-")[-1]) - # fov, i.e. tiles - cycle_info["fov"].append(acq_info[5].split("-")[-1]) - # exposure - exposure = cycle_info["exposure"].append( - acq_info[6].split("-")[-1].strip(".tif") +def cycle_info(data_folder: Path, ref_marker: str, antigen_cycle_no: int = cycle_no): + if "AntigenCycle" in data_folder.name: + cycle = "AntigenCycle" + elif "BleachCycle" in data_folder.name: + cycle = "BleachCycle" + else: + raise ValueError( + "Folder name should contain either AntigenCycle or BleachCycle." ) - info = pd.DataFrame(cycle_info) + full_image_path = list(data_folder.glob("*.tif")) + images = [x.name for x in full_image_path] + + # Define the patterns + b_pattern = r"_B-(\d+)" # What is this? + rack_pattern = r"_R-(\d+)" + well_pattern = r"_W-(\d+)" + roi_pattern = r"_G-(\d+)" + fov_pattern = r"_F-(\d+)" + exposure_pattern = r"_E-(\d+)" + filter_pattern = r".*_([^_]*)_\d+bit" + + marker_pattern = rf"{cycle}_(.*?)_" + marker_values = extract_values( + pattern=marker_pattern, strings=images, number_cast=False + ) + filter_values = extract_values( + pattern=filter_pattern, strings=images, number_cast=False + ) - markers = info["marker"].unique() - markers_subset = np.setdiff1d(markers, [ref_marker]) - # insert the exposure level colum at the end of the info dataframe - info.insert(len(cycle_info), "exposure_level", np.zeros(info.shape[0])) - # label the exposure level of the reference markers with "ref" - info.loc[info["marker"] == ref_marker, "exposure_level"] = "ref" + # Todo: Is this even needed? + filter_values = [ + ref_marker if marker == ref_marker else value + for marker, value in zip(marker_values, filter_values) + ] - for m in markers_subset: - exposure = info.loc[info["marker"] == m]["exposure"].unique() - val = pd.to_numeric(exposure) - val = np.sort(val) - for level, value in enumerate(val): - info.loc[ - (info["marker"] == m) & (info["exposure"] == str(value)), - "exposure_level", - ] = level + 1 - - # change the data type to numeric - info["rack"] = pd.to_numeric(info["rack"], downcast="unsigned") - info["well"] = pd.to_numeric(info["well"], downcast="unsigned") - info["roi"] = pd.to_numeric(info["roi"], downcast="unsigned") - info["fov"] = pd.to_numeric(info["fov"], downcast="unsigned") - info["exposure"] = pd.to_numeric(info["exposure"], downcast="unsigned") + info = pd.DataFrame( + { + "img_full_path": full_image_path, + "image": images, + "marker": extract_values( + pattern=marker_pattern, strings=images, number_cast=False + ), + "filter": filter_values, + "rack": extract_values(pattern=rack_pattern, strings=images), + "well": extract_values(pattern=well_pattern, strings=images), + "roi": extract_values(pattern=roi_pattern, strings=images), + "fov": extract_values(pattern=fov_pattern, strings=images), + "exposure": extract_values(pattern=exposure_pattern, strings=images), + } + ) - return info + markers = info["marker"].unique() + markers_subset = np.setdiff1d(markers, [ref_marker]) + # insert the exposure level colum at the end of the info dataframe + info.insert(loc=info.shape[1], column="exposure_level", value=0) -def bleach_cycle_info( - bleach_folder=bleach_dir, ref_marker=ref_marker, antigen_cycle_no=cycle_no -): - bleach_cycle = f"{antigen_cycle_no-1:03d}" - - images = [x for x in os.listdir(bleach_folder) if x.endswith(".tif")] - - cycle_info = { - "img_full_path": [], - "image": images, - "marker": [], - "filter": [], - "rack": [], - "well": [], - "roi": [], - "fov": [], - "exposure": [], - } + info["exposure_level"] = ( + info.groupby("marker")["exposure"].rank(method="dense").astype(int) + ) + info.loc[info["marker"] == ref_marker, "exposure_level"] = "ref" - for im in images: - marker_info = im.split("BleachCycle")[-1].split("_") - acq_info = im.split("sensor")[-1].split("_") - # add the information to the cycle_info dictionary - # img full path - cycle_info["img_full_path"].append(os.path.join(bleach_folder, im)) - # marker and fluorophore - # marker_info=['', 'DAPI', 'V0', 'DAPI', '16bit', 'M-20x-S Fluor full sensor', 'B-1', 'R-1', 'W-2', 'G-1', 'F-10', 'E-16.0.tif'] - m = marker_info[1] - if m == ref_marker: - cycle_info["marker"].append(ref_marker) - else: - cycle_info["marker"].append( - "_".join([bleach_cycle, "bleach", marker_info[3]]) - ) - cycle_info["filter"].append(marker_info[3]) - # rack - cycle_info["rack"].append(acq_info[2].split("-")[-1]) - # well - cycle_info["well"].append(acq_info[3].split("-")[-1]) - # roi - cycle_info["roi"].append(acq_info[4].split("-")[-1]) - # fov, i.e. tiles - cycle_info["fov"].append(acq_info[5].split("-")[-1]) - # exposure - exposure = cycle_info["exposure"].append( - acq_info[6].split("-")[-1].strip(".tif") + if cycle == "BleachCycle": + bleach_cycle = f"{antigen_cycle_no - 1:03d}" + info.loc[info["marker"] == "None", "marker"] = ( + f"{bleach_cycle}_bleach_" + info["filter"] ) - info = pd.DataFrame(cycle_info) - - markers = info["filter"].unique() - markers_subset = np.setdiff1d(markers, [ref_marker]) - info.insert(len(cycle_info), "exposure_level", np.zeros(info.shape[0])) - info.loc[info["marker"] == ref_marker, "exposure_level"] = "ref" - # info.to_csv(os.path.join(cycles_path,'test_bleach.csv')) - - for m in markers_subset: - exposure = info.loc[info["filter"] == m]["exposure"].unique() - val = pd.to_numeric(exposure) - val = np.sort(val) - for level, value in enumerate(val): - info.loc[ - (info["filter"] == m) & (info["exposure"] == str(value)), - "exposure_level", - ] = level + 1 - - info["rack"] = pd.to_numeric(info["rack"], downcast="unsigned") - info["well"] = pd.to_numeric(info["well"], downcast="unsigned") - info["roi"] = pd.to_numeric(info["roi"], downcast="unsigned") - info["fov"] = pd.to_numeric(info["fov"], downcast="unsigned") - info["exposure"] = pd.to_numeric(info["exposure"], downcast="unsigned") - return info -def create_dir(dir_path): - if not os.path.exists(dir_path): - os.mkdir(dir_path) - - def create_ome(img_info, xy_tile_positions_units, img_path): img_name = img_info["name"] device_name = img_info["device"] @@ -241,7 +169,7 @@ def create_ome(img_info, xy_tile_positions_units, img_path): sig_bits = img_info["sig_bits"] if pixel_units == "mm": pixel_size = 1000 * pixel_size - # pixel_units='um' + # pixel_units="um" no_of_tiles = len(xy_tile_positions_units) tifff.tiffcomment(img_path, "") # UUID=ome_types.model.tiff_data.UUID(file_name=img_name,value=uuid4().urn) @@ -344,8 +272,6 @@ def setup_coords(x, y, pix_units): def tile_position(metadata_string): - # meta_dict = {TAGS[key] : img.tag[key] for key in img.tag_v2} - # ome=BeautifulSoup(meta_dict['ImageDescription'][0],'xml') ome = BeautifulSoup(metadata_string, "xml") x = float(ome.StageLabel["X"]) y = float(ome.StageLabel["Y"]) @@ -353,7 +279,7 @@ def tile_position(metadata_string): pixel_size = float(ome.Pixels["PhysicalSizeX"]) pixel_units = ome.Pixels["PhysicalSizeXUnit"] bit_depth = ome.Pixels["Type"] - significantBits = int(ome.Pixels["SignificantBits"]) + significant_bits = int(ome.Pixels["SignificantBits"]) return { "x": x, "y": y, @@ -361,7 +287,7 @@ def tile_position(metadata_string): "pixel_size": pixel_size, "pixel_units": pixel_units, "bit_depth": bit_depth, - "sig_bits": significantBits, + "sig_bits": significant_bits, } @@ -389,6 +315,7 @@ def create_stack( dtype_ref = img_ref.dtype ref_marker = list(info.loc[info["exposure_level"] == "ref", "marker"])[0] markers = info["marker"].unique() + markers_subset = np.setdiff1d(markers, [ref_marker]) sorted_markers = np.insert(markers_subset, 0, ref_marker) sorted_filters = [ @@ -421,7 +348,7 @@ def create_stack( fov_id = np.sort(fov_id) stack_name = "cycle-{C}-{prefix}-exp-{E}-rack-{R}-well-{W}-roi-{ROI}-{T}-{M}.{img_format}".format( - C=f"{(offset+antigen_cycle_no):03d}", + C=f"{(offset + antigen_cycle_no):03d}", prefix=cycle_prefix, E=exp_level, R=r, @@ -443,6 +370,7 @@ def create_stack( "exposure", ].tolist()[0] exposure_per_marker = [exp] + for s in markers_subset: exp = groupB.loc[ (groupB["marker"] == s) @@ -507,8 +435,11 @@ def create_stack( def main(): - antigen_info = antigen_cycle_info(antigen_folder=antigen_dir) - bleach_info = bleach_cycle_info(bleach_folder=bleach_dir) + antigen_info = cycle_info(data_folder=antigen_dir, ref_marker=ref_marker) + bleach_info = cycle_info(data_folder=bleach_dir, ref_marker=ref_marker) + print(antigen_info) + sys.exit() + exp = antigen_info["exposure_level"].unique() exp = exp[exp != "ref"] exp.sort() From 46749c3ee3418ebe043e7066b55a5f089b88e70a Mon Sep 17 00:00:00 2001 From: mbruhns Date: Fri, 22 Mar 2024 16:17:50 +0100 Subject: [PATCH 03/11] Reformat. --- nfmacsima.py | 69 ++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/nfmacsima.py b/nfmacsima.py index 5defb93..66c85cf 100644 --- a/nfmacsima.py +++ b/nfmacsima.py @@ -84,7 +84,9 @@ def extract_values(pattern, strings, number_cast=True): ] -def cycle_info(data_folder: Path, ref_marker: str, antigen_cycle_no: int = cycle_no): +def cycle_info( + data_folder: Path, reference_marker: str, antigen_cycle_no: int = cycle_no +): if "AntigenCycle" in data_folder.name: cycle = "AntigenCycle" elif "BleachCycle" in data_folder.name: @@ -116,7 +118,7 @@ def cycle_info(data_folder: Path, ref_marker: str, antigen_cycle_no: int = cycle # Todo: Is this even needed? filter_values = [ - ref_marker if marker == ref_marker else value + reference_marker if marker == reference_marker else value for marker, value in zip(marker_values, filter_values) ] @@ -136,16 +138,13 @@ def cycle_info(data_folder: Path, ref_marker: str, antigen_cycle_no: int = cycle } ) - markers = info["marker"].unique() - markers_subset = np.setdiff1d(markers, [ref_marker]) - # insert the exposure level colum at the end of the info dataframe info.insert(loc=info.shape[1], column="exposure_level", value=0) info["exposure_level"] = ( info.groupby("marker")["exposure"].rank(method="dense").astype(int) ) - info.loc[info["marker"] == ref_marker, "exposure_level"] = "ref" + info.loc[info["marker"] == reference_marker, "exposure_level"] = "ref" if cycle == "BleachCycle": bleach_cycle = f"{antigen_cycle_no - 1:03d}" @@ -261,6 +260,7 @@ def create_ome(img_info, xy_tile_positions_units, img_path): def setup_coords(x, y, pix_units): + # Todo: This only gets executed if pix_units is mm. Check if this is the case. if pix_units == "mm": x_norm = 1000 * (np.array(x) - np.min(x)) # /pixel_size y_norm = 1000 * (np.array(y) - np.min(y)) # /pixel_size @@ -333,56 +333,47 @@ def create_stack( for w in wells: well_no = "well_{n}".format(n=f"{w:02d}") output_levels.append(well_no) - groupA = info.loc[(info["rack"] == r) & (info["well"] == w)] - rois = groupA["roi"].unique() + group_a = info.loc[(info["rack"] == r) & (info["well"] == w)] + rois = group_a["roi"].unique() for roi in rois: roi_no = "roi_{n}".format(n=f"{roi:02d}") exp_level_no = "exp_level_{n}".format(n=f"{exp_level:02d}") output_levels.extend((roi_no, exp_level_no)) counter = 0 - groupB = groupA.loc[groupA["roi"] == roi] - A = groupB.loc[(groupB["exposure_level"] == "ref")] + group_b = group_a.loc[group_a["roi"] == roi] + A = group_b.loc[(group_b["exposure_level"] == "ref")] stack_size_z = A.shape[0] * len(markers) - fov_id = groupB.loc[groupB["marker"] == ref_marker, "fov"].unique() + fov_id = group_b.loc[group_b["marker"] == ref_marker, "fov"].unique() fov_id = np.sort(fov_id) - stack_name = "cycle-{C}-{prefix}-exp-{E}-rack-{R}-well-{W}-roi-{ROI}-{T}-{M}.{img_format}".format( - C=f"{(offset + antigen_cycle_no):03d}", - prefix=cycle_prefix, - E=exp_level, - R=r, - W=w, - ROI=roi, - T=target_name, # markers or filters - M=target, - img_format="ome.tiff", - ) + stack_name = f"""cycle-{(offset + antigen_cycle_no):03d}-{cycle_prefix}-exp-{exp_level}- + rack-{r}-well-{w}-roi-{roi}-{target_name}-{target}.ome.tiff""" X = [] Y = [] stack = np.zeros((stack_size_z, height, width), dtype=dtype_ref) - exp = groupB.loc[ - (groupB["marker"] == ref_marker) - & (groupB["fov"] == 1) - & (groupB["exposure_level"] == "ref"), + exp = group_b.loc[ + (group_b["marker"] == ref_marker) + & (group_b["fov"] == 1) + & (group_b["exposure_level"] == "ref"), "exposure", ].tolist()[0] exposure_per_marker = [exp] for s in markers_subset: - exp = groupB.loc[ - (groupB["marker"] == s) - & (groupB["fov"] == 1) - & (groupB["exposure_level"] == exp_level), + exp = group_b.loc[ + (group_b["marker"] == s) + & (group_b["fov"] == 1) + & (group_b["exposure_level"] == exp_level), "exposure", ].tolist()[0] exposure_per_marker.append(exp) for tile in fov_id: - if img_ref := groupB.loc[ - (groupB["marker"] == ref_marker) & (groupB["fov"] == tile), + if img_ref := group_b.loc[ + (group_b["marker"] == ref_marker) & (group_b["fov"] == tile), "img_full_path", ].tolist(): with tifff.TiffFile(img_ref[0]) as tif: @@ -396,10 +387,10 @@ def create_stack( counter += 1 for m in markers_subset: - img_marker = groupB.loc[ - (groupB["marker"] == m) - & (groupB["fov"] == tile) - & (groupB["exposure_level"] == exp_level), + img_marker = group_b.loc[ + (group_b["marker"] == m) + & (group_b["fov"] == tile) + & (group_b["exposure_level"] == exp_level), "img_full_path", ].tolist()[0] img = tifff.imread(img_marker) @@ -435,10 +426,8 @@ def create_stack( def main(): - antigen_info = cycle_info(data_folder=antigen_dir, ref_marker=ref_marker) - bleach_info = cycle_info(data_folder=bleach_dir, ref_marker=ref_marker) - print(antigen_info) - sys.exit() + antigen_info = cycle_info(data_folder=antigen_dir, reference_marker=ref_marker) + bleach_info = cycle_info(data_folder=bleach_dir, reference_marker=ref_marker) exp = antigen_info["exposure_level"].unique() exp = exp[exp != "ref"] From 4d34e535919132b6de5f09ca920f9d31a05cbd48 Mon Sep 17 00:00:00 2001 From: Victor Perez Meza Date: Thu, 23 May 2024 09:27:33 +0200 Subject: [PATCH 04/11] replaced docker-publish.yml with new version --- .github/workflows/docker-publish.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 58ec527..4c76205 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,4 +1,4 @@ -name: publish docker container for macsima2mc.py +name: Docker # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by @@ -7,9 +7,9 @@ name: publish docker container for macsima2mc.py on: [push] #release: - # types: published + # types: published #pull_request: - # branches: [ "main" ] + # branches: [ "main" ] env: # Use docker.io for Docker Hub if empty @@ -30,19 +30,16 @@ jobs: id-token: write steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Checkout repository uses: actions/checkout@v3 # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - - name: Install cosign + - name: Install Cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + uses: sigstore/cosign-installer@v3.1.1 with: - cosign-release: 'v1.11.0' + cosign-release: 'v2.2.1' # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx @@ -82,4 +79,4 @@ jobs: platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} From f4ade80b2e73fad557e12f028475f9e846e154a1 Mon Sep 17 00:00:00 2001 From: VictorDidier <68292394+VictorDidier@users.noreply.github.com> Date: Thu, 23 May 2024 09:29:28 +0200 Subject: [PATCH 05/11] Updated docker-publish.yml --- .github/workflows/docker-publish.yml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 58ec527..3c465ad 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,15 +1,15 @@ -name: publish docker container for macsima2mc.py +name: docker for macsima2mc # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by # separate terms of service, privacy policy, and support # documentation. -on: [push] - #release: - # types: published - #pull_request: - # branches: [ "main" ] +on: + release: + types: published + pull_request: + branches: [ "main" ] env: # Use docker.io for Docker Hub if empty @@ -30,19 +30,16 @@ jobs: id-token: write steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Checkout repository uses: actions/checkout@v3 # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - - name: Install cosign + - name: Install Cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + uses: sigstore/cosign-installer@v3.1.1 with: - cosign-release: 'v1.11.0' + cosign-release: 'v2.2.1' # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx @@ -82,4 +79,4 @@ jobs: platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} From cac069408c04d8cb74f40fef1bb42cf5a2f28af9 Mon Sep 17 00:00:00 2001 From: VictorDidier <68292394+VictorDidier@users.noreply.github.com> Date: Thu, 23 May 2024 09:30:04 +0200 Subject: [PATCH 06/11] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3c465ad..fbf2125 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,11 +5,11 @@ name: docker for macsima2mc # separate terms of service, privacy policy, and support # documentation. -on: - release: - types: published - pull_request: - branches: [ "main" ] +on: [push] + #release: + # types: published + #pull_request: + # branches: [ "main" ] env: # Use docker.io for Docker Hub if empty From 54c7dfbdf5413102709f412c3735c05b03d15f1a Mon Sep 17 00:00:00 2001 From: VictorDidier <68292394+VictorDidier@users.noreply.github.com> Date: Fri, 24 May 2024 11:08:37 +0200 Subject: [PATCH 07/11] Update docker-publish to checkout_v4.yml --- .github/workflows/docker-publish.yml | 62 +++++++++++++++++----------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4c76205..c146ede 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -6,10 +6,14 @@ name: Docker # documentation. on: [push] - #release: - # types: published - #pull_request: - # branches: [ "main" ] +# schedule: +# - cron: '1 * * * *' +# push: +# branches: [ "main" ] + # Publish semver tags as releases. +# tags: [ 'v*.*.*' ] +# pull_request: +# branches: [ "main" ] env: # Use docker.io for Docker Hub if empty @@ -31,25 +35,27 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - - name: Install Cosign + - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v3.1.1 + uses: sigstore/cosign-installer@v3.5.0 #v3.1.1 with: - cosign-release: 'v2.2.1' - - # Workaround: https://github.com/docker/build-push-action/issues/461 - - name: Setup Docker buildx - uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 # Login against a Docker registry except on PR # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} if: github.event_name != 'pull_request' - uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -59,24 +65,34 @@ jobs: # https://github.com/docker/metadata-action - name: Extract Docker metadata id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Get the tag name - run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - + # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 with: context: . - platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} From 9569b4dde48aa91940ca36e69082a6c51b3f287a Mon Sep 17 00:00:00 2001 From: Victor Perez Meza Date: Fri, 24 May 2024 11:18:09 +0200 Subject: [PATCH 08/11] changed publish-docker --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a030930..36ba9e1 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer From 5262077dbeec99571c0d7a3a5a34883cfc07c456 Mon Sep 17 00:00:00 2001 From: Victor Perez Meza Date: Fri, 24 May 2024 11:21:42 +0200 Subject: [PATCH 09/11] add platforms line to the publish docker --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1d75641..f51079c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -76,6 +76,7 @@ jobs: uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 with: context: . + platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 1ccd843dc624285b05d91372cec9592c9fc1e96f Mon Sep 17 00:00:00 2001 From: Victor Perez Meza Date: Fri, 24 May 2024 15:20:02 +0200 Subject: [PATCH 10/11] deleted multiple print commands --- macsima2mc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/macsima2mc.py b/macsima2mc.py index 98ecd31..e64f0c7 100644 --- a/macsima2mc.py +++ b/macsima2mc.py @@ -470,7 +470,6 @@ def create_stack(info,exp_level=1, groupA=info.loc[(info['rack']==r) & (info['well']==w)] rois=groupA['roi'].unique() - print("remember to delete the line below line 467") rois=[1] for roi in rois: roi_no='roi_{n}'.format(n=f'{roi:02d}') @@ -514,8 +513,7 @@ def create_stack(info,exp_level=1, exp=groupB.loc[(groupB['marker']==ref_marker) & (groupB['fov']==1) & (groupB['exposure_level']=='ref'),'exposure'].tolist()[0] exposure_per_marker.append(exp) for s in markers_subset: - #print(s) - #print("test",groupB.loc[(groupB['marker']==s) & (groupB['fov']==1) & (groupB['exposure_level']==exp_level),'exposure'].tolist()) + exp=groupB.loc[(groupB['marker']==s) & (groupB['fov']==1) & (groupB['exposure_level']==exp_level),'exposure'].tolist()[0] exposure_per_marker.append(exp) From 9a51b6fda3388ea7e80790e055fd63a47c1b17c9 Mon Sep 17 00:00:00 2001 From: Victor Perez Meza Date: Fri, 24 May 2024 15:25:47 +0200 Subject: [PATCH 11/11] add -rr flag to arguments in macsima2mc.py --- macsima2mc.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/macsima2mc.py b/macsima2mc.py index e64f0c7..0095f5c 100644 --- a/macsima2mc.py +++ b/macsima2mc.py @@ -72,6 +72,12 @@ only the antigen images will be extracted.' ) +parser.add_argument('-rr', + '--remove_reference_marker', + action='store_true', + help='Set up in the markers.csv file the removal of the reference markers in all cycles except for the first one.' + ) + args=parser.parse_args() #---END_CLI-BLOCK---# @@ -668,8 +674,18 @@ def main(): df=pd.DataFrame(out_ant).groupby('exposure_level').get_group(e) df.drop('exposure_level',axis=1,inplace=True) df.insert(0,'channel_number',list(range(1,1+df.shape[0]))) + + if args.remove_reference_marker: + dna_indices=df.loc[df['marker_name']=='DAPI'].index.values + for n,i in enumerate(dna_indices): + if n>0: + df.at[i,'remove']='TRUE' + df.to_csv(stack_path / 'markers_exp_{level}.csv'.format(level=e),index=False) + + + if __name__=='__main__': main()