NIMS_to_BIDS_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NIMS-to-BIDS converter v.2\n",
    "Written by Natalia Vélez, 9/17\n",
    "\n",
    "Changes in this version:\n",
    "\n",
    "* Change to file structure: raw data are now stored in `$PI_SCRATCH`\n",
    "* Incorporate changes to protocol file, including tracking sequence numbers \n",
    "* Add support for fieldmaps\n",
    "* Remove redundancies in the code\n",
    "\n",
    "Load dependencies:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing Libraries...\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import unicode_literals\n",
    "from __future__ import division\n",
    "from __future__ import absolute_import\n",
    "\n",
    "from builtins import input\n",
    "from builtins import open\n",
    "from builtins import str\n",
    "from future import standard_library\n",
    "standard_library.install_aliases()\n",
    "print(\"Importing Libraries...\\n\")\n",
    "\n",
    "import warnings\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "from shutil import copyfile\n",
    "import json\n",
    "import sys\n",
    "import subprocess\n",
    "import glob\n",
    "from os.path import join as opj # Helper function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open and write to file\n",
    "def write_file(contents, path):\n",
    "    with open(path, 'w') as openfile:\n",
    "        openfile.write(contents)\n",
    "        \n",
    "# Make directory (if it doesn't already exist)\n",
    "def mkdir(path):\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "\n",
    "# Open and write to JSON file\n",
    "def write_json(data, path):\n",
    "    json_data = json.dumps(data)\n",
    "    write_file(json_data, path)\n",
    "    \n",
    "# Replaces basename in path (ignoring text in the path name)\n",
    "# e.g. IN: replace_basename('fieldmap_path/fieldmap.nii.gz', 'fieldmap', 'magnitude') \n",
    "# OUT: 'fieldmap_path/magnitude.nii.gz'\n",
    "def replace_basename(series, oldstr, newstr):\n",
    "    d = series.apply(os.path.dirname)\n",
    "    f = series.apply(os.path.basename)\n",
    "    f = f.str.replace(oldstr, newstr)\n",
    "    \n",
    "    df = pd.DataFrame(dict(d = d, f = f))\n",
    "    new_path_fun = lambda row: opj(row['d'], row['f'])\n",
    "    new_paths = df.apply(new_path_fun, axis = 1).tolist()\n",
    "        \n",
    "    return new_paths\n",
    "\n",
    "# Assembles BIDS filename\n",
    "def assemble_bids_filename(d, custom_keys = None):\n",
    "    default_keys = ['participant', 'ses', 'task', 'acq', 'run', 'echo', 'filetype']\n",
    "    key_order = default_keys if custom_keys is None else custom_keys\n",
    "    \n",
    "    filename_parts = [d[k] for k in key_order if k in d.keys()]\n",
    "    filename_base = '_'.join(filename_parts)\n",
    "    filename = filename_base + '.nii.gz'\n",
    "    \n",
    "    return filename"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set input and output directories:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating report at: /share/PI/hyo/SwiSt/reports/NIMS2BIDS_SwiSt_20171004_0155_report.txt\n",
      "Input: /scratch/PI/hyo/SwiSt/NIMS_data\n",
      "Output: /share/PI/hyo/SwiSt/BIDS_data\n",
      "BIDS_info file: /share/PI/hyo/SwiSt/BIDS_info.xlsx\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "60"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "home_dir = os.environ['PI_HOME']\n",
    "scratch_dir = os.environ['PI_SCRATCH']\n",
    "\n",
    "#project_name =  str(sys.argv[1]).strip(' ') # Uncomment for production\n",
    "project_name = 'SwiSt'\n",
    "project_dir = opj(home_dir, project_name)\n",
    "report_dir = opj(project_dir, 'reports')\n",
    "NIMS = opj(scratch_dir, project_name, 'NIMS_data')\n",
    "BIDS = opj(project_dir, 'BIDS_data')\n",
    "BIDS_file = glob.glob(opj(project_dir, '*BIDS_info*.xlsx'))\n",
    "\n",
    "#Make sure there's only one bids file\n",
    "assert len(BIDS_file) == 1, 'This folder does not have a BIDS_info file or it has more than one info file' \n",
    "xls = pd.ExcelFile(BIDS_file[0])\n",
    "\n",
    "# Write to report\n",
    "mkdir(report_dir)\n",
    "mkdir(BIDS)\n",
    "\n",
    "# Create a new text file to report outputs\n",
    "report_tstamp = '{:%Y%m%d_%H%M}'.format(datetime.datetime.now())\n",
    "report_path = opj(report_dir, 'NIMS2BIDS_%s_%s_report.txt' % (project_name, report_tstamp))\n",
    "copyjob_path = opj(report_dir, 'NIMS2BIDS_%s_%s_copyjob.csv' % (project_name, report_tstamp))\n",
    "report_file = open(report_path, 'w')\n",
    "\n",
    "def report_print(msg):\n",
    "    report_file.write(msg + '\\n')\n",
    "    print(msg)\n",
    "    \n",
    "# Write to report\n",
    "print('Creating report at: %s' % report_path)\n",
    "report_file.write('=== NIMS TO BIDS CONVERSION ===\\n')\n",
    "report_file.write('Timestamp: {:%Y-%m-%d %H:%M:%S}\\n'.format(datetime.datetime.now()))\n",
    "report_print('Input: %s' % NIMS)\n",
    "report_print('Output: %s' % BIDS)\n",
    "print('BIDS_info file: %s' % BIDS_file[0])\n",
    "report_file.write('BIDS_info file: %s \\n \\n -----' % BIDS_file[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load dataset description:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = xls.parse('dataset').iloc[1,:]\n",
    "dataset = dataset.dropna() # Remove blank fields\n",
    "dataset_data = {'Name': 'Dataset', 'BIDSVersion': '1.0.0'} # Default, required arguments\n",
    "dataset_custom = dataset.to_dict()\n",
    "dataset_data.update(dataset_custom) # Override defaults\n",
    "\n",
    "# Format authors as array\n",
    "if 'Authors' in dataset_data:\n",
    "    dataset_data['Authors'] = dataset_data['Authors'].split(',')\n",
    "\n",
    "# Save to file\n",
    "dataset_file = opj(BIDS, 'dataset_description.json')\n",
    "dataset_data\n",
    "with open(dataset_file, 'w') as openfile:\n",
    "    json.dump(dataset_data, openfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load participant information:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "participants = xls.parse('participants')\n",
    "participants.participant_id = ['sub-%02d' % int(n) for n in participants.participant_id]\n",
    "\n",
    "# Save to file\n",
    "participants_file = opj(BIDS, 'participants.tsv')\n",
    "participants.to_csv(participants_file, index = False, sep = '\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load task data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = xls.parse('tasks').iloc[1:,]\n",
    "\n",
    "# Save tasks to file\n",
    "for task in tasks.iterrows():\n",
    "    task_dict = task[1].to_dict()\n",
    "    del task_dict['BIDS_scan_title']\n",
    "    \n",
    "    task_fname = opj(BIDS, '%s_bold.json' % task[1]['BIDS_scan_title'])\n",
    "    write_json(task_dict, task_fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load protocol data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "protocol = xls.parse('protocol', convert_float=False).iloc[1:,]\n",
    "protocol = protocol[~pd.isnull(protocol.sequence_type)] # Remove columns with missing BIDS data types"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find input (NIMS-formatted) files and specify output (BIDS-formatted) files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy-job successfully assembled! Details at: /share/PI/hyo/SwiSt/reports/NIMS2BIDS_SwiSt_20171004_0155_copyjob.csv\n"
     ]
    }
   ],
   "source": [
    "report_file.write('Assembling copy job:')\n",
    "session_IDs = participants.nims_title\n",
    "participant_IDs = participants.participant_id\n",
    "custom_protocols = np.unique(protocol.nims_title)\n",
    "custom_protocols = custom_protocols[custom_protocols != 'default']\n",
    "copy_job_cols = ['session', 'in_img', 'out_img', 'out_info', 'out_info_file']\n",
    "copy_job = pd.DataFrame(columns = copy_job_cols)\n",
    "\n",
    "# Helper function: Searches for matching input files\n",
    "def input_path(row, session_id):\n",
    "    d = row.to_dict()\n",
    "    \n",
    "    # Templates: Build search string based on sequence number and type\n",
    "    input_fname = '*fieldmap.nii.gz' if d['sequence_type'] == 'fmap' else '*_1.nii.gz'\n",
    "    input_template = opj(NIMS, session_id, '*_%i_1_%s', input_fname)\n",
    "    input_search = input_template % (d['sequence_no'], d['NIMS_scan_title'])\n",
    "    \n",
    "    # Matches: Find matching sequences\n",
    "    input_matches = glob.glob(input_search)    \n",
    "    path = input_matches[0] if input_matches else np.nan\n",
    "    if not input_matches:\n",
    "        report_print('Missing file: %s' % input_search)\n",
    "        \n",
    "    return path\n",
    "    \n",
    "# Helper function: Builds path for output files\n",
    "def output_path(row, participant_id):\n",
    "    d = row.to_dict()\n",
    "    filetypes = {'func': 'bold', 'fmap': 'fieldmap'}\n",
    "    out_d = {\n",
    "        'participant': participant_id,\n",
    "        'task': None if pd.isnull(d['BIDS_scan_title']) else d['BIDS_scan_title'],\n",
    "        'run': 'run-%02d' % d['run_number'] if ~np.isnan(d['run_number']) else None,\n",
    "        'filetype': filetypes[d['sequence_type']] if d['sequence_type'] in filetypes.keys() else None}\n",
    "    out_d = {k:v for k,v in out_d.items() if v is not None}\n",
    "    \n",
    "    out_file = assemble_bids_filename(out_d)\n",
    "    out_path = opj(BIDS, participant_id, d['sequence_type'], out_file)\n",
    "    \n",
    "    return out_path\n",
    "    \n",
    "# Helper function: Prepares JSON file keys\n",
    "def output_keys(row, participant_id, session_protocol):\n",
    "    # Fields common to all sequences\n",
    "    standard_fields = ['nims_title', 'sequence_no', 'NIMS_scan_title',\n",
    "                       'BIDS_scan_title', 'run_number', 'sequence_type']\n",
    "    \n",
    "    # Remove standard fields and NA'sfrom row \n",
    "    # (only custom fields related to the current sequence remain)\n",
    "    row = row.drop(standard_fields)\n",
    "    row = row.dropna()\n",
    "    row_dict = row.to_dict()\n",
    "    \n",
    "    # If dictionary contains an IntendedFor field (for fieldmaps), replace the sequence numbers\n",
    "    # with BIDS-formatted filenames\n",
    "    if 'IntendedFor' in row_dict:\n",
    "        # Subject directory\n",
    "        subj_dir = opj(BIDS, participant_id)\n",
    "        \n",
    "        if isinstance(row_dict['IntendedFor'], str):\n",
    "            target_runs_raw = row_dict['IntendedFor'].split(' ')\n",
    "            target_runs = [int(r) for r in target_runs_raw]\n",
    "        else:\n",
    "            target_runs = [int(row_dict['IntendedFor'])]\n",
    "        target_protocol = session_protocol[session_protocol['sequence_no'].isin(target_runs)]\n",
    "\n",
    "        # Get BIDS output for each file\n",
    "        get_target_path = lambda row: output_path(row, participant_id)\n",
    "        get_rel_path = lambda path: os.path.relpath(path, subj_dir)\n",
    "        target_full_path = target_protocol.apply(get_target_path, axis = 1)\n",
    "        target_full_path = target_full_path.tolist()\n",
    "        target_path = [get_rel_path(path) for path in target_full_path]\n",
    "        \n",
    "        # Replace IntendedFor with properly formatted paths\n",
    "        row_dict['IntendedFor'] = target_path\n",
    "        \n",
    "    # Convert row_dict to JSON string\n",
    "    row_json = json.dumps(row_dict) if row_dict else np.nan\n",
    "    return row_json\n",
    "    \n",
    "# Iterate over participants and assemble copy job:\n",
    "for session, participant_id in zip(session_IDs, participant_IDs):\n",
    "    #participant_id = participants[participants.nims_title == session]['participant_id']\n",
    "    \n",
    "    # Get correct protocol\n",
    "    is_custom = session in custom_protocols\n",
    "    protocol_type = 'CUSTOM' if is_custom else 'DEFAULT'\n",
    "    protocol_ref = session if is_custom else 'default'\n",
    "    session_protocol = protocol[protocol.nims_title == protocol_ref]\n",
    "    \n",
    "    # Assemble copy_job\n",
    "    input_files = session_protocol.apply(lambda row: input_path(row, session), axis=1)\n",
    "    output_files = session_protocol.apply(lambda row: output_path(row, participant_id), axis=1)\n",
    "    output_info = session_protocol.apply(lambda row: output_keys(row, participant_id, session_protocol), axis=1)\n",
    "    output_info_files = [f.replace('.nii.gz', '.json') for f in output_files]\n",
    "    output_info_files = [f if isinstance(info, str) else np.nan for f,info in zip(output_info_files, output_info)]\n",
    "    session_col = [session for _ in range(len(input_files))]\n",
    "    \n",
    "    # Convert to list\n",
    "    input_files = input_files.tolist()\n",
    "    output_files = output_files.tolist()\n",
    "    output_info = output_info.tolist()\n",
    "    \n",
    "    tmp_items = list(zip(copy_job_cols, [session_col, input_files, output_files, output_info, output_info_files]))\n",
    "    tmp_copy = pd.DataFrame.from_items(tmp_items)\n",
    "    copy_job = copy_job.append(tmp_copy)\n",
    "    \n",
    "    # Add magnitude images to fieldmaps\n",
    "    mag_images = copy_job[copy_job['out_img'].str.contains('fmap')].copy()\n",
    "    mag_images['in_img'] = replace_basename(mag_images['in_img'], 'fieldmap', '')\n",
    "    mag_images['out_img'] = replace_basename(mag_images['out_img'], 'fieldmap', 'magnitude')\n",
    "\n",
    "    mag_images['out_info'] = np.nan\n",
    "    mag_images['out_info_file'] = np.nan\n",
    "    copy_job = copy_job.append(mag_images)\n",
    "    \n",
    "# Save copy job to reports\n",
    "copy_job.to_csv(copyjob_path)\n",
    "\n",
    "# Raise an error if files are missing\n",
    "if np.any(pd.isnull(copy_job['in_img'])):\n",
    "    report_file.write('Conversion not successful: Missing files\\n')\n",
    "    raise Exception('ERROR: Missing files found. Please consult report for details.')\n",
    "else:\n",
    "    copyjob_msg = 'Copy-job successfully assembled! Details at: %s' % copyjob_path\n",
    "    print(copyjob_msg)\n",
    "    report_file.write(copyjob_msg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The variable `copy_job` contains a dataframe with: input images (`in_img`), output images (`out_img`), output metadata (`out_info`), metadata path (`out_info_file`)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that all files have been found, let's make all of the necessary folders:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New directories created:\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-04\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-05\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-06\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-07\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-08\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-09\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-10\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-11\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-12\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-04/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-05/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-06/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-07/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-08/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-09/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-10/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-11/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-12/anat\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-04/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-05/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-06/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-07/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-08/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-09/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-10/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-11/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-12/fmap\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-04/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-05/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-06/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-07/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-08/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-09/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-10/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-11/func\n",
      "/share/PI/hyo/SwiSt/BIDS_data/sub-12/func\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_types = np.unique(protocol['sequence_type'].tolist())\n",
    "sub_dirs = [opj(BIDS, sub) for sub in participant_IDs]\n",
    "data_dirs = [opj(s, d) for d in data_types for s in sub_dirs]\n",
    "new_dirs = sub_dirs + data_dirs\n",
    "\n",
    "report_print('New directories created:')\n",
    "for d in new_dirs:\n",
    "    if not os.path.exists(d):\n",
    "        report_print(d)\n",
    "        os.makedirs(d)\n",
    "report_file.write('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copy over the files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying files...\n"
     ]
    }
   ],
   "source": [
    "report_print('Copying files...')\n",
    "for idx, row in copy_job.iterrows():\n",
    "    report_file.write('Input: %s\\n' % row['in_img'])\n",
    "    report_file.write('Output: %s \\n \\n' % row['out_img'])\n",
    "    copyfile(row['in_img'], row['out_img'])\n",
    "    #os.system('fslreorient2std %s %s' % (row['out_img'], row['out_img']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create metadata:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done! Unpacking successful.\n"
     ]
    }
   ],
   "source": [
    "copy_metadata = copy_job.dropna()\n",
    "for row in copy_metadata.iterrows():\n",
    "    write_file(row[1]['out_info'], row[1]['out_info_file'])\n",
    "    \n",
    "report_print('Done! Unpacking successful.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}