From c3491ed12bc9ea4cc2817edbbe79b10fa8454a44 Mon Sep 17 00:00:00 2001 From: Kelvin Rodriguez Date: Thu, 18 Jan 2018 16:29:18 -0700 Subject: [PATCH] added env, updated params, cleanup #1 --- config.yml | 7 ++- environment.yml | 164 ++++++++++++++++++++++++++++++++++++++++++++++++ matching.py | 68 +++++++++++++------- 3 files changed, 214 insertions(+), 25 deletions(-) create mode 100644 environment.yml diff --git a/config.yml b/config.yml index 48a49e6..8207685 100644 --- a/config.yml +++ b/config.yml @@ -1,4 +1,5 @@ -davbin : "/nfs/software/davinci_install/share/davinci/library/bin/" -themisroot: "/home/krodriguez/jay/day/I01210009_I00849001/" -outpath : "/data/out" +davinci_bin : "/nfs/software/davinci_install/share/davinci/library/bin/" +themis_root: "/home/krodriguez/jay/day/I01210009_I00849001/" +inpath : "/data/themis" +outpath : "/data/themis" diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..a964e60 --- /dev/null +++ b/environment.yml @@ -0,0 +1,164 @@ +name: themis +channels: +- menpo +- jlaura +- defaults +- conda-forge +- krodriguez +- usgs-astrogeology +dependencies: +- appnope=0.1.0=py36_0 +- backports=1.0=py36_1 +- backports.functools_lru_cache=1.4=py36_1 +- blas=1.1=openblas +- bleach=2.0.0=py36_0 +- boost=1.65.1=py36_0 +- boost-cpp=1.65.1=1 +- bzip2=1.0.6=1 +- ca-certificates=2017.11.5=0 +- cairo=1.14.6=5 +- certifi=2017.11.5=py36_0 +- click=6.7=py_1 +- click-plugins=1.0.3=py36_0 +- cligj=0.4.0=py36_0 +- curl=7.55.1=0 +- cycler=0.10.0=py36_0 +- cython=0.27.3=py36_0 +- descartes=1.1.0=py36_0 +- dill=0.2.7.1=py36_0 +- entrypoints=0.2.3=py36_1 +- expat=2.1.0=3 +- ffmpeg=3.2.4=3 +- fontconfig=2.12.1=6 +- freetype=2.7=2 +- freexl=1.0.4=0 +- gdal=2.2.3=py36_0 +- geopandas=0.3.0=py36_0 +- geos=3.6.2=1 +- geotiff=1.4.2=0 +- gettext=0.19.8.1=0 +- giflib=5.1.4=0 +- glib=2.51.4=0 +- harfbuzz=1.3.4=2 +- hdf4=4.2.13=0 +- hdf5=1.10.1=1 +- html5lib=1.0.1=py_0 +- icu=58.2=0 +- ipykernel=4.7.0=py36_0 +- ipython=6.2.1=py36_1 +- ipython_genutils=0.2.0=py36_0 +- jasper=1.900.1=4 +- jedi=0.10.2=py36_0 +- jinja2=2.10=py36_0 +- jpeg=9b=2 +- json-c=0.12.1=0 +- jsonschema=2.6.0=py36_0 +- jupyter_client=5.2.1=py36_0 +- jupyter_core=4.4.0=py_0 +- kealib=1.4.7=4 +- krb5=1.14.2=0 +- libdap4=3.18.3=2 +- libffi=3.2.1=3 +- libgdal=2.2.3=1 +- libgfortran=3.0.0=0 +- libiconv=1.15=0 +- libnetcdf=4.4.1.1=10 +- libpng=1.6.28=2 +- libpq=9.6.3=0 +- libsodium=1.0.15=1 +- libspatialindex=1.8.5=1 +- libspatialite=4.3.0a=18 +- libssh2=1.8.0=2 +- libtiff=4.0.7=1 +- libwebp=0.5.2=7 +- libxml2=2.9.7=0 +- markupsafe=1.0=py36_0 +- matplotlib=2.1.1=py36_0 +- mistune=0.8.3=py_0 +- munch=2.2.0=py36_0 +- nbconvert=5.3.1=py_1 +- nbformat=4.4.0=py36_0 +- ncurses=5.9=10 +- networkx=2.0=py36_1 +- numexpr=2.6.4=py36_0 +- numpy=1.14.0=py36_blas_openblas_200 +- openblas=0.2.20=7 +- opencv=3.3.0=py36_blas_openblas_203 +- openjpeg=2.3.0=1 +- openssl=1.0.2n=0 +- pandas=0.22.0=py36_0 +- pandoc=2.1=0 +- pandocfilters=1.4.1=py36_0 +- pcre=8.39=0 +- pexpect=4.3.1=py36_0 +- pickleshare=0.7.4=py36_0 +- pillow=4.3.0=py36_1 +- pip=9.0.1=py36_1 +- pixman=0.34.0=1 +- poppler=0.61.1=2 +- poppler-data=0.4.8=0 +- proj4=4.9.3=5 +- prompt_toolkit=1.0.15=py36_0 +- psycopg2=2.7.3.2=py36_0 +- ptyprocess=0.5.2=py36_0 +- pvl=0.3.0=py36_0 +- pygments=2.2.0=py36_0 +- pyparsing=2.2.0=py36_0 +- pyproj=1.9.5.1=py36_0 +- pyqt=5.6.0=py36_4 +- python=3.6.3=1 +- python-dateutil=2.6.1=py36_0 +- pytz=2017.3=py_2 +- pyyaml=3.12=py36_1 +- pyzmq=16.0.2=py36_3 +- qt=5.6.2=h9e3eb04_4 +- readline=6.2=0 +- rtree=0.8.3=py36_0 +- runipy=0.1.5=py36_1 +- scikit-learn=0.19.1=py36_blas_openblas_201 +- scipy=1.0.0=py36_blas_openblas_201 +- setuptools=38.4.0=py36_0 +- shapely=1.6.3=py36_0 +- simplegeneric=0.8.1=py36_0 +- sip=4.18=py36_1 +- six=1.11.0=py36_1 +- sqlite=3.13.0=1 +- testpath=0.3.1=py36_0 +- tk=8.5.19=2 +- tornado=4.5.3=py36_0 +- traitlets=4.3.2=py36_0 +- vlfeat=0.9.20=1 +- wcwidth=0.1.7=py36_0 +- webencodings=0.5=py36_0 +- wheel=0.30.0=py36_2 +- x264=20131217=3 +- xerces-c=3.2.0=0 +- xz=5.2.3=0 +- zeromq=4.2.1=1 +- zlib=1.2.11=0 +- affine=2.1.0=pyh128a3a6_1 +- decorator=4.1.2=py36h69a1b52_0 +- fiona=1.7.10=py36h42caba7_0 +- h5py=2.7.1=py36h39cdac5_0 +- libboost=1.65.1=hcc95346_4 +- libcxx=4.0.1=h579ed51_0 +- libcxxabi=4.0.1=hebd6815_0 +- libkml=1.3.0=h694af70_2 +- libprotobuf=3.4.1=h326466f_0 +- olefile=0.44=py36ha08bf50_0 +- protobuf=3.4.1=py36h314970b_0 +- pysal=1.14.2=py36h48a81df_1 +- sqlalchemy=1.2.0=py36h1de35cc_0 +- yaml=0.1.7=hc338f04_2 +- cyvlfeat=0.4.6=py36_0 +- plio=0.1.0=py36_0 +- pip: + - autocnet==0.1.0 + - backports.functools-lru-cache==1.4 + - ipython-genutils==0.2.0 + - jupyter-client==5.2.1 + - jupyter-core==4.4.0 + - prompt-toolkit==1.0.15 + - pysis==0.6.0 +prefix: /Users/krodriguez/anaconda3/envs/themis + diff --git a/matching.py b/matching.py index 9b9252c..2ded240 100644 --- a/matching.py +++ b/matching.py @@ -1,7 +1,13 @@ +# Deal with annoying numpy warning +import warnings +warnings.filterwarnings("ignore") + from shapely import wkt import glob import os +import argparse + import autocnet from autocnet import CandidateGraph from autocnet.graph.edge import Edge @@ -23,7 +29,6 @@ import numpy as np from plio.io.io_gdal import GeoDataset -# Hack to get ISIS setup in a Jupyterhub environment running as a root service import subprocess from subprocess import Popen, PIPE @@ -33,6 +38,7 @@ from plio.date import marstime from collections import OrderedDict +import yaml record = OrderedDict({ 'file_id1' : '', @@ -62,38 +68,54 @@ 'diff_stddev' : 0, }) +def run_davinci(script, infile, outfile, root_dir, args=[]): + command = ['davinci', '-f', '{}{}'.format(root_dir, script), 'from={}'.format(infile), 'to={}'.format(outfile)] + if args: + command.extend(args) + print(' '.join(command)) + p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, err = p.communicate(b"input data that is passed to subprocess' stdin") + rc = p.returncode + + if rc != 0: + raise Exception('Davinci returned non-zero error code {} : {}'.format(rc, err.decode('utf-8'))) + return output.decode('utf-8'), err.decode('utf-8') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('config', action='store', help='path to config file', default='config.yaml') + parser.add_argument('id1', action='store', help='Image ID for image one') + parser.add_argument('id2', action='store', help='Image ID for image two') + parser.add_argument('-c', '--config', action='store', help='path to config file', default='./config.yml') + parser.add_argument('-v', '--verbose', action='store_true', help='Activates verbose output', default=False) + parser.add_argument('-g', '--graph', action='store_true', help='Display graphs of the data as is goes through.', default=False) args = parser.parse_args().__dict__ - with open(arg['config'], 'r') as ymlfile: + with open(args['config'], 'r') as ymlfile: cfg = yaml.load(ymlfile) - # Glob the dir for a file list of the lvl 1 - root_dir = cfg['dpath'] + if args['verbose']: + def verboseprint(*args): + # Print each argument separately so caller doesn't need to + # stuff everything to be printed into a single string + import pprint + pp = pprint.PrettyPrinter(indent=2) + for arg in args: + pp.pprint(arg), + print() + else: + verboseprint = lambda *a: None # do-nothing function - def run_davinci(script, infile, outfile, args=[]): - command = ['davinci', '-f', '{}{}'.format(root_dir, script), 'from={}'.format(infile), 'to={}'.format(outfile)] - if args: - command.extend(args) - print(' '.join(command)) - p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, err = p.communicate(b"input data that is passed to subprocess' stdin") - rc = p.returncode + verboseprint('ARGS:', args) + verboseprint('CONFIG:', cfg) - if rc != 0: - raise Exception('Davinci returned non-zero error code {} : {}'.format(rc, err.decode('utf-8'))) - return output.decode('utf-8'), err.decode('utf-8') + # patch in davinci_bin onto run_davinci to prevent contstantly having to pass it in + run_davinci = lambda x, y, z=[]: run_davinci(x,y,cfg['davinci_bin'], z) - files = glob.glob(os.path.join(root_dir, '*lev1.cub')) - - filid1 = os.path.basename(files[0].split('.')[0]) - filid2 = os.path.basename(files[1].split('.')[0]) + # Glob the dir for a file list of the lvl 1 + root_dir = cfg['inpath'] - record['file_id1'] = filid1 - record['file_id2'] = filid2 + record['id1'] = args['id1'] + record['id2'] = args['id2'] out, err = run_davinci('thm_pre_process.dv', filid1, '{}.lev1.cub'.format(filid1)) out, err = run_davinci('thm_pre_process.dv', filid2, '{}.lev1.cub'.format(filid2)) @@ -124,8 +146,10 @@ def run_davinci(script, infile, outfile, args=[]): print("STDOUT:", e.stdout.decode('utf-8')) print("STDERR:", e.stderr.decode('utf-8')) + ############################################################# cg = CandidateGraph.from_filelist(files) + # The range of DN values over the data is small, so the threshold for differentiating interesting features must be small. cg.extract_features(extractor_parameters={'contrastThreshold':0.0000000001}) cg.match()