NIMS_to_BIDS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import unicode_literals\n",
    "from __future__ import division\n",
    "from __future__ import absolute_import\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing Libraries...\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from builtins import input\n",
    "from builtins import open\n",
    "from builtins import str\n",
    "from future import standard_library\n",
    "standard_library.install_aliases()\n",
    "print(\"Importing Libraries...\\n\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "from shutil import copyfile\n",
    "import json\n",
    "import sys\n",
    "import subprocess\n",
    "import pdb # Debugging\n",
    "import glob\n",
    "from os.path import join as opj # Helper function\n",
    "\n",
    "#home_dir = os.environ['PI_HOME']\n",
    "home_dir = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NIMS_to_BIDS.py can take the project's file path as an argument\n",
      "No argument detected\n",
      "Please drag in file path from folder\n",
      "/Users/mdclark/Desktop/NIMS_to_BIDS\n",
      "['BIDS_info.xlsx']\n"
     ]
    }
   ],
   "source": [
    "#Get Data Filepath\n",
    "if (len(sys.argv) == 2):\n",
    "    project_dir = str(sys.argv[1]).strip(' ')\n",
    "else:\n",
    "    print(\"NIMS_to_BIDS.py can take the project's file path as an argument\\nNo argument detected\\nPlease drag in file path from folder\")\n",
    "    project_dir = input().strip(' ')\n",
    "\n",
    "project_filepath = opj(home_dir, project_dir)\n",
    "\n",
    "#path variables\n",
    "BIDS= opj(project_filepath, 'BIDS_data')\n",
    "NIMS= opj(project_filepath, 'NIMS_data')\n",
    "\n",
    "#Read files\n",
    "\n",
    "#Figure out what bids info xlsx is named \n",
    "project_file_contents = os.listdir(project_filepath)\n",
    "BIDS_filename = [file for file in project_file_contents if \"BIDS_info\" in file]\n",
    "print(BIDS_filename)\n",
    "\n",
    "#Make sure there's only one bids file\n",
    "assert len(BIDS_filename) == 1, 'This folder does not have a BIDS_info file or it has more than one info file' \n",
    "\n",
    "xls = pd.ExcelFile(project_filepath + \"/\" + BIDS_filename[0])\n",
    "\n",
    "#Make folder if folder doesn't exist function\n",
    "def makefolder(name):\n",
    "    if not os.path.exists(name):\n",
    "        os.makedirs(name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Load and Clean XLS File\n",
    "participants = xls.parse('participants')\n",
    "participants.participant_id = ['%02d' % int(n) for n in participants.participant_id]\n",
    "\n",
    "protocol = xls.parse('protocol', convert_float=False).iloc[1:,:6] #columns 5 on are reference columns\n",
    "protocol = protocol.dropna(axis=0, thresh=3) #get rid of items that don't have a bids equivalent\n",
    "protocol.run_str = ['_run-%02d' % n if not np.isnan(n) else '_' for n in protocol.run_number]\n",
    "\n",
    "#fieldmap = xls.parse('fieldmap', convert_float=False)\n",
    "#fieldmap.intended_for = [str(s) for s in fieldmap.intended_for]\n",
    "#fieldmap.intended_for = [s.split() for s in fieldmap.intended_for]\n",
    "\n",
    "#Create \"bold\" portion of filename\n",
    "protocol['bold_filename'] = ''\n",
    "protocol.loc[protocol['ANAT_or_FUNC'] == 'func', 'bold_filename'] = '_bold'\n",
    "\n",
    "#Concatanate filepath and clean\n",
    "protocol.subj_dirs = opj(BIDS, 'subj-###')\n",
    "protocol[\"BIDS_scan_title_path\"] = BIDS + \"/sub-###/\" + protocol.ANAT_or_FUNC + \"/sub-###_\" + protocol.BIDS_scan_title + protocol.run_str + protocol.bold_filename + \".nii.gz\"\n",
    "#protocol.BIDS_scan_title_path = protocol.BIDS_scan_title_path.str.replace('_run-nan', '') #For items that don't have runs\n",
    "\n",
    "#Create list for NIMS -> bids conversion\n",
    "NIMS_protocol_filenames = protocol.NIMS_scan_title.tolist() #Convert protocol scan titles to list\n",
    "NIMS_BIDS_conversion = protocol[[\"NIMS_scan_title\",\"BIDS_scan_title_path\"]]\n",
    "#pdb.set_trace()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def check_against_protocol(participants,protocol): \n",
    "    \n",
    "    all_files_correct = True\n",
    "    \n",
    "    for index, row in participants.iterrows():\n",
    "\n",
    "        try:\n",
    "            #pdb.set_trace()\n",
    "            NIMS_participant = opj(NIMS, row.nims_title)\n",
    "            NIMS_participant_subdirs = [opj(NIMS_participant, d) for d in os.listdir(NIMS_participant)]\n",
    "            NIMS_participant_subdirs = [d for d in NIMS_participant_subdirs if os.path.isdir(d)]\n",
    "\n",
    "            for item in set(NIMS_protocol_filenames):\n",
    "                protocol_dirs = [d for d in NIMS_participant_subdirs if item in d]\n",
    "                directory_filenames = []\n",
    "                for d in protocol_dirs:\n",
    "                    protocol_search = glob.glob(opj(d, \"*.nii.gz\"))\n",
    "                    directory_filenames.append(protocol_search[0])\n",
    "\n",
    "                protocol_filenames = NIMS_BIDS_conversion[NIMS_BIDS_conversion.NIMS_scan_title.str.contains(item)]\n",
    "                protocol_filenames = protocol_filenames.iloc[:,1].tolist()\n",
    "\n",
    "                if len(directory_filenames) < len(protocol_filenames):\n",
    "                    print('{} : sub-{} : << {} {} files in folder {} files in protocol\\n'.                    format(str(row.nims_title), str(row.participant_id), item.rjust(20), len(directory_filenames), len(protocol_filenames)))\n",
    "\n",
    "                elif len(directory_filenames) > len(protocol_filenames):\n",
    "                    print('{} : sub-{} : >> {} {} files in folder {} files in protocol\\n'.                    format(str(row.nims_title), str(row.participant_id), item.rjust(20), len(directory_filenames), len(protocol_filenames)))\n",
    "                    all_files_correct = False\n",
    "                    \n",
    "                elif len(directory_filenames) == len(protocol_filenames):\n",
    "                    print('{} : sub-{} : == {} {} files in folder {} files in protocol\\n'.                    format(str(row.nims_title), str(row.participant_id), item.rjust(20), len(directory_filenames), len(protocol_filenames)))\n",
    "\n",
    "            print(\"------------\")\n",
    "        \n",
    "        except:\n",
    "            all_files_correct = False\n",
    "            print(\"sub-\" + str(row.participant_id) + \" : -- ERROR - folder is missing \\n------------\")\n",
    "\n",
    "\n",
    "    return all_files_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def write_text_files(participants, protocol): \n",
    "    \n",
    "    def to_file(filename, content):\n",
    "        file_path = opj(BIDS, filename + '.json') \n",
    "        with open(file_path, \"w\") as text_file:\n",
    "            text_file.write(content)\n",
    "    \n",
    "    #Data Description\n",
    "    dataset_description = json.dumps({\"BIDSVersion\": \"1.0.0\",                                    \"License\": \"\",                                    \"Name\": \"dummy task name\",                                   \"ReferencesAndLinks\": \"\"})\n",
    "    to_file(str(\"dataset_description\"), str(dataset_description))\n",
    "    \n",
    "\n",
    "    #Task Description\n",
    "    for item in set(protocol.loc[protocol.ANAT_or_FUNC == \"func\", 'BIDS_scan_title']):\n",
    "        full_task_name = protocol.loc[protocol.BIDS_scan_title == item, 'full_task_name']\n",
    "        full_task_name = full_task_name.reset_index(drop=True)[0] #Gets first instance of RT\n",
    "        \n",
    "        repetition_time = protocol.loc[protocol.BIDS_scan_title == item, 'repetition_time']\n",
    "        repetition_time = repetition_time.reset_index(drop=True)[0] #Gets first instance of RT\n",
    "        task_json = json.dumps({\"RepetitionTime\": repetition_time, \"TaskName\" : full_task_name})\n",
    "\n",
    "        to_file(str(item + \"_bold\"), str(task_json))\n",
    "\n",
    "    #TSV\n",
    "    participant_tsv = participants.loc[:, ['participant_id', 'sex', 'age']]\n",
    "    participant_tsv.loc[:, 'participant_id'] = \"sub-\" + participant_tsv.loc[:, 'participant_id'].apply(str)\n",
    "    #Had to write csv and then change it due to python 2/3 incompatability\n",
    "    participant_tsv.to_csv(opj(BIDS, 'participants.tsv'), index=False)\n",
    "    # Read in the file\n",
    "    with open(opj(BIDS, 'participants.tsv'), 'r') as file :\n",
    "        filedata = file.read()\n",
    "\n",
    "    # Replace the target string\n",
    "    filedata = filedata.replace(',', '\\t')\n",
    "\n",
    "    # Write the file out again\n",
    "    with open(opj(BIDS, 'participants.tsv'), 'w') as file:\n",
    "        file.write(filedata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convert_to_bids(participants, protocol):\n",
    "    \n",
    "    print(\"Comparing Folders to Protocol...\\n\")\n",
    "    \n",
    "    if check_against_protocol(participants,protocol): #Function returns true is everything matches\n",
    "        \n",
    "        print(\"Creating BIDS_data folder\\n\")\n",
    "        #Make BIDS Folder\n",
    "        makefolder(BIDS)\n",
    "        participants.participant_id.apply(lambda x: makefolder(opj(BIDS, 'sub-' + str(x), 'anat')))\n",
    "        participants.participant_id.apply(lambda x: makefolder(opj(BIDS, 'sub-' + str(x), 'func')))\n",
    "        \n",
    "        for index, row in participants.iterrows():\n",
    "            NIMS_participant = opj(NIMS, row.nims_title)\n",
    "            NIMS_participant_subdirs = [opj(NIMS_participant, d) for d in os.listdir(NIMS_participant)]\n",
    "            NIMS_participant_subdirs = [d for d in NIMS_participant_subdirs if os.path.isdir(d)]\n",
    "\n",
    "            for item in set(NIMS_protocol_filenames):\n",
    "                print(item)\n",
    "                #if item == 'T1w_9mm_sag':\n",
    "                    #pdb.set_trace()\n",
    "                protocol_dirs = [d for d in NIMS_participant_subdirs if item in d]\n",
    "                directory_filenames = []\n",
    "                for d in protocol_dirs:\n",
    "                    protocol_search = glob.glob(opj(d, \"*.nii.gz\"))\n",
    "                    directory_filenames.append(protocol_search[0])\n",
    "\n",
    "                protocol_filenames = NIMS_BIDS_conversion[NIMS_BIDS_conversion.NIMS_scan_title.str.contains(item)]\n",
    "                protocol_filenames = protocol_filenames.iloc[:,1].tolist()\n",
    "\n",
    "                for index, item in enumerate(directory_filenames):\n",
    "                    oldpath = directory_filenames[index]\n",
    "                    newpath = (protocol_filenames[index].replace(\"###\", str(row.participant_id)))\n",
    "                    copyfile(oldpath, newpath)\n",
    "\n",
    "                    print(\"sub-\" + str(row.participant_id) + \": ++ \"+ os.path.basename(newpath).rjust(20))\n",
    "            print(\"------------\")\n",
    "\n",
    "        print(\"\\nCreating JSON and .tsv Files\")\n",
    "        \n",
    "        write_text_files(participants, protocol)\n",
    "       \n",
    "        print(\"\\nDone!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Comparing Folders to Protocol...\n",
      "\n",
      "20170609_1904 : sub-222 : ==        T1w_9mm_BRAVO 1 files in folder 1 files in protocol\n",
      "\n",
      "20170609_1904 : sub-222 : ==   BOLD_EPI_29mm_2sec 4 files in folder 4 files in protocol\n",
      "\n",
      "------------\n",
      "20170612_1741 : sub-270 : ==        T1w_9mm_BRAVO 1 files in folder 1 files in protocol\n",
      "\n",
      "20170612_1741 : sub-270 : ==   BOLD_EPI_29mm_2sec 4 files in folder 4 files in protocol\n",
      "\n",
      "------------\n",
      "20170610_1201 : sub-256 : ==        T1w_9mm_BRAVO 1 files in folder 1 files in protocol\n",
      "\n",
      "20170610_1201 : sub-256 : ==   BOLD_EPI_29mm_2sec 4 files in folder 4 files in protocol\n",
      "\n",
      "------------\n",
      "Creating BIDS_data folder\n",
      "\n",
      "T1w_9mm_BRAVO\n",
      "sub-222: ++  sub-222_T1w_.nii.gz\n",
      "BOLD_EPI_29mm_2sec\n",
      "sub-222: ++ sub-222_task-emp_run-01_bold.nii.gz\n",
      "sub-222: ++ sub-222_task-emp_run-02_bold.nii.gz\n",
      "sub-222: ++ sub-222_task-emp_run-03_bold.nii.gz\n",
      "sub-222: ++ sub-222_task-emp_run-04_bold.nii.gz\n",
      "------------\n",
      "T1w_9mm_BRAVO\n",
      "sub-270: ++  sub-270_T1w_.nii.gz\n",
      "BOLD_EPI_29mm_2sec\n",
      "sub-270: ++ sub-270_task-emp_run-01_bold.nii.gz\n",
      "sub-270: ++ sub-270_task-emp_run-02_bold.nii.gz\n",
      "sub-270: ++ sub-270_task-emp_run-03_bold.nii.gz\n",
      "sub-270: ++ sub-270_task-emp_run-04_bold.nii.gz\n",
      "------------\n",
      "T1w_9mm_BRAVO\n",
      "sub-256: ++  sub-256_T1w_.nii.gz\n",
      "BOLD_EPI_29mm_2sec\n",
      "sub-256: ++ sub-256_task-emp_run-01_bold.nii.gz\n",
      "sub-256: ++ sub-256_task-emp_run-02_bold.nii.gz\n",
      "sub-256: ++ sub-256_task-emp_run-03_bold.nii.gz\n",
      "sub-256: ++ sub-256_task-emp_run-04_bold.nii.gz\n",
      "------------\n",
      "\n",
      "Creating JSON and .tsv Files\n",
      "\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "convert_to_bids(participants, protocol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}