Skip to content

Commit

Permalink
move read_csv and write_csv from AMs to sasbase. clear some temp file…
Browse files Browse the repository at this point in the history 
…refs when done with them
  • Loading branch information
@tomweber-sas
tomweber-sas committed Jul 19, 2023
1 parent 6184edf commit 3649d02
Show file tree
Hide file tree
Showing 7 changed files with 60 additions and 247 deletions.
37 changes: 34 additions & 3 deletions saspy/sasbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1178,7 +1178,22 @@ def read_csv(self, file: str, table: str = '_csv', libref: str = '', results: st
if results == '':
results = self.results

self._io.read_csv(file, table, libref, self.nosub, opts)
code = "filename _x "

if file.lower().startswith("http"):
code += "url "

code += "\""+file+"\";\n"
code += "proc import datafile=_x out="
if len(libref):
code += libref+"."
code += "'"+table.strip().replace("'", "''")+"'n dbms=csv replace; "+self._impopts(opts)+" run;"
code += "filename _x clear;\n"

if self.nosub:
print(code)
else:
ll = self._io.submit(code, "text")

if self.exist(table, libref):
sd = SASdata(self, libref, table, results)
Expand Down Expand Up @@ -1230,9 +1245,25 @@ def write_csv(self, file: str, table: str, libref: str = '',
:return: SAS log
"""
dsopts = dsopts if dsopts is not None else {}
opts = opts if opts is not None else {}
opts = opts if opts is not None else {}

code = "filename _x \""+file+"\";\n"
code += "options nosource;\n"
code += "proc export data="

if len(libref):
code += libref+"."

code += "'"+table.strip().replace("'", "''")+"'n "+self._dsopts(dsopts)+" outfile=_x dbms=csv replace;\n"
code += self._expopts(opts)+" run;\n"
code += "filename _x clear;"
code += "options source;\n"

if self.nosub:
print(code)
else:
log = self._io.submit(code, "text")['LOG']

log = self._io.write_csv(file, table, libref, self.nosub, dsopts, opts)
if not self.batch:
print(log)
else:
Expand Down
52 changes: 0 additions & 52 deletions saspy/sasiocom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
#

import datetime
import csv
import io
import numbers
import os
Expand Down Expand Up @@ -601,57 +600,6 @@ def read_sasdata(self, table: str, libref: str=None, dsopts: dict=None) -> tuple

return (header, rows, meta)

def read_csv(self, filepath: str, table: str, libref: str=None, nosub: bool=False, opts: dict=None):
"""
Submit an import job to the SAS workspace.
:param filepath [str]: File URI.
:param table [str]: Table name.
:option libref [str]: Library name.
:option nosob [bool]: Return the SAS code instead of executing it.
:option opts [dict]: SAS PROC IMPORT options.
"""
opts = opts if opts is not None else {}
filepath = 'url ' + filepath if filepath.lower().startswith('http') else filepath
tablepath = self._tablepath(table, libref=libref)

proc_code = """
filename csv_file "{}";
proc import datafile=csv_file out={} dbms=csv replace;
{}
run;
""".format(filepath.replace('"', '""'), tablepath, self._sb._impopts(opts))

if nosub is True:
return proc_code
else:
return self.submit(proc_code, 'text')

def write_csv(self, filepath: str, table: str, libref: str=None, nosub: bool=True, dsopts: dict=None, opts: dict=None):
"""
Submit an export job to the SAS workspace.
:param filepath [str]: File URI.
:param table [str]: Table name.
:option libref [str]: Library name.
:option nosob [bool]: Return the SAS code instead of executing it.
:option opts [dict]: SAS PROC IMPORT options.
:option dsopts [dict]: SAS dataset options.
"""
opts = opts if opts is not None else {}
dsopts = dsopts if dsopts is not None else {}
tablepath = self._tablepath(table, libref=libref)

proc_code = """
filename csv_file "{}";
proc export data={} {} outfile=csv_file dbms=csv replace;
{}
run;
""".format(filepath.replace('"', '""'), tablepath, self._sb._dsopts(dsopts), self._sb._expopts(opts))

if nosub is True:
return proc_code
else:
return self.submit(proc_code, 'text')['LOG']

def dataframe2sasdata(self, df: '<Pandas Data Frame object>', table: str ='a',
libref: str ="", keep_outer_quotes: bool=False,
embedded_newlines: bool=True,
Expand Down
66 changes: 10 additions & 56 deletions saspy/sasiohttp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,8 +546,8 @@ def __init__(self, session, **kwargs):
if self.ctx == {}:
raise SASHTTPconnectionError(msg="No context information returned for context {}\n{}".format(self.ctxname, contexts))
else:
self.ctx = contexts
self.ctxname = 'tom'
self.ctx = contexts
self.ctxname = self.serverid

return

Expand Down Expand Up @@ -604,8 +604,6 @@ def _authenticate(self, user, pw, authcode, client_id, client_secret, jwt):
return js

def _get_contexts(self):
#import pdb; pdb.set_trace()

# GET Contexts
conn = self.HTTPConn; conn.connect()

Expand Down Expand Up @@ -1266,55 +1264,6 @@ def exist(self, table: str, libref: str ="") -> bool:
return exists
"""

def read_csv(self, file: str, table: str, libref: str ="", nosub: bool=False, opts: dict ={}) -> '<SASdata object>':
'''
This method will import a csv file into a SAS Data Set and return the SASdata object referring to it.
file - eithe the OS filesystem path of the file, or HTTP://... for a url accessible file
table - the name of the SAS Data Set to create
libref - the libref for the SAS Data Set being created. Defaults to WORK, or USER if assigned
opts - a dictionary containing any of the following Proc Import options(datarow, delimiter, getnames, guessingrows)
'''
code = "filename x "

if file.lower().startswith("http"):
code += "url "

code += "\""+file+"\";\n"
code += "proc import datafile=x out="
if len(libref):
code += libref+"."
code += "'"+table.strip().replace("'", "''")+"'n dbms=csv replace; "+self._sb._impopts(opts)+" run;"

if nosub:
print(code)
else:
ll = self.submit(code, "text")

def write_csv(self, file: str, table: str, libref: str ="", nosub: bool =False, dsopts: dict ={}, opts: dict ={}) -> 'The LOG showing the results of the step':
'''
This method will export a SAS Data Set to a file in CCSV format.
file - the OS filesystem path of the file to be created (exported from the SAS Data Set)
table - the name of the SAS Data Set you want to export to a CSV file
libref - the libref for the SAS Data Set.
opts - a dictionary containing any of the following Proc Export options(delimiter, putnames)
'''
code = "filename x \""+file+"\";\n"
code += "options nosource;\n"
code += "proc export data="

if len(libref):
code += libref+"."

code += "'"+table.strip().replace("'", "''")+"'n "+self._sb._dsopts(dsopts)+" outfile=x dbms=csv replace; "
code += self._sb._expopts(opts)+" run\n;"
code += "options source;\n"

if nosub:
print(code)
else:
ll = self.submit(code, "text")
return ll['LOG']

def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permission: str = '', **kwargs):
"""
This method uploads a local file to the SAS servers file system.
Expand Down Expand Up @@ -1862,6 +1811,7 @@ def sasdata2dataframeCSV(self, table: str, libref: str ='', dsopts: dict =None,
code += "proc export data=work.sasdata2dataframe outfile=_tomodsx dbms=csv replace;\n"
code += self._sb._expopts(opts)+" run;\n"
code += "proc delete data=work.sasdata2dataframe(memtype=view);run;\n"
code += "filename _tomodsx;"

ll = self.submit(code, 'text')
logf = ll['LOG']
Expand Down Expand Up @@ -1891,7 +1841,9 @@ def sasdata2dataframeCSV(self, table: str, libref: str ='', dsopts: dict =None,
if varcat[i] in self._sb.sas_date_fmts + self._sb.sas_time_fmts + self._sb.sas_datetime_fmts:
df[dvarlist[i]] = pd.to_datetime(df[dvarlist[i]], errors='coerce')

ll = self.submit("filename _sp_updn;", 'text')
code = "data _null_; fdelete(_sp_updn); run;\nfilename _sp_updn;"

ll = self.submit(code, 'text')
logf += ll['LOG']

return df
Expand Down Expand Up @@ -2065,7 +2017,7 @@ def sasdata2dataframeDISK(self, table: str, libref: str ='', dsopts: dict = None
code += " '"+varlist[i]+"'n "
if i % 10 == 9:
code +='\n'
code += rdelim+";\nrun;"
code += rdelim+";\nrun;\nfilename _tomodsx;"

ll = self.submit(code, "text")

Expand Down Expand Up @@ -2112,7 +2064,9 @@ def sasdata2dataframeDISK(self, table: str, libref: str ='', dsopts: dict = None
if varcat[i] in self._sb.sas_date_fmts + self._sb.sas_time_fmts + self._sb.sas_datetime_fmts:
df[dvarlist[i]] = pd.to_datetime(df[dvarlist[i]], errors='coerce')

ll = self.submit("filename _sp_updn;", 'text')
code = "data _null_; fdelete(_sp_updn); run;\nfilename _sp_updn;"

ll = self.submit(code, 'text')
logf += ll['LOG']

return df
Expand Down
57 changes: 2 additions & 55 deletions saspy/sasioiom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1187,61 +1187,6 @@ def exist(self, table: str, libref: str ="") -> bool:

return bool(exists)

def read_csv(self, file: str, table: str, libref: str ="", nosub: bool =False, opts: dict = None) -> '<SASdata object>':
"""
This method will import a csv file into a SAS Data Set and return the SASdata object referring to it.
file - eithe the OS filesystem path of the file, or HTTP://... for a url accessible file
table - the name of the SAS Data Set to create
libref - the libref for the SAS Data Set being created. Defaults to WORK, or USER if assigned
opts - a dictionary containing any of the following Proc Import options(datarow, delimiter, getnames, guessingrows)
"""
opts = opts if opts is not None else {}

code = "filename x "

if file.lower().startswith("http"):
code += "url "

code += "\""+file+"\";\n"
code += "proc import datafile=x out="
if len(libref):
code += libref+"."
code += "'"+table.strip().replace("'", "''")+"'n dbms=csv replace; "+self._sb._impopts(opts)+" run;"

if nosub:
print(code)
else:
ll = self.submit(code, "text")

def write_csv(self, file: str, table: str, libref: str ="", nosub: bool =False, dsopts: dict = None, opts: dict = None) -> 'The LOG showing the results of the step':
"""
This method will export a SAS Data Set to a file in CSV format.
file - the OS filesystem path of the file to be created (exported from the SAS Data Set)
table - the name of the SAS Data Set you want to export to a CSV file
libref - the libref for the SAS Data Set.
dsopts - a dictionary containing any of the following SAS data set options(where, drop, keep, obs, firstobs)
opts - a dictionary containing any of the following Proc Export options(delimiter, putnames)
"""
dsopts = dsopts if dsopts is not None else {}
opts = opts if opts is not None else {}

code = "filename x \""+file+"\";\n"
code += "options nosource;\n"
code += "proc export data="

if len(libref):
code += libref+"."

code += "'"+table.strip().replace("'", "''")+"'n "+self._sb._dsopts(dsopts)+" outfile=x dbms=csv replace; "
code += self._sb._expopts(opts)+" run\n;"
code += "options source;\n"

if nosub:
print(code)
else:
ll = self.submit(code, "text")
return ll['LOG']

def upload_slow(self, localfile: str, remotefile: str, overwrite: bool = True, permission: str = '', **kwargs):
"""
This method uploads a local file to the SAS servers file system.
Expand Down Expand Up @@ -1894,6 +1839,8 @@ def sasdata2dataframeCSV(self, table: str, libref: str ='', dsopts: dict = None,
code += "proc export data=work.sasdata2dataframe outfile="+outname+" dbms=csv replace;\n"
code += self._sb._expopts(opts)+" run;\n"
code += "proc delete data=work.sasdata2dataframe(memtype=view);run;\n"
if local:
code += "filename _tomodsx;"

ll = self._asubmit(code, 'text')

Expand Down
Loading

0 comments on commit 3649d02

Please sign in to comment.