-
Notifications
You must be signed in to change notification settings - Fork 0
/
data_zipper.py
50 lines (47 loc) · 1.98 KB
/
data_zipper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import os
import re
import subprocess as sp
import time
import process_helper as ph
import glob
#NOTE: For some reason the zipping halts with this funciton.
# I used it just to print the commands so I could copy paste
# those to prompt....
skip_processed = True
skip_original = False
if( not skip_processed):
processes = ph.ProcessCounter(3)
stats = ['max','min','mean']
variables = ['SSS', 'SST', 'SBS', 'icethic', 'icecon', 'snowthic', 'icevvelo', 'iceuvelo', 'sorunoff']
in_dir = '/scratch/project_2001635/siiriasi/smartsea_data/daily_test/'
for var in variables:
for stat in stats:
# print("Handling {}, {}".format(var,stat))
out_dir = '/scratch/project_2001635/siiriasi/smartsea_data/daily_test/{}/'.format(stat)
# the_command = "zip -v {}/{}_monthly_{}.zip {}".format(\
# out_dir, var, \
# stat, \
# " ".join(glob.glob("{}/{}*.nc".format(out_dir,var))))
the_command = "zip -v {}/{}_monthly_{}.zip {}".format(\
out_dir, var, \
stat, \
" {}/{}*.nc".format(out_dir,var))
print(the_command)
# processes.add(the_command)
processes.wait_all()
if(not skip_original):
stats = ['vmax','vmin','vmean']
# variables = ['SSS', 'SST', 'SBS', 'icethic', 'icecon', 'snowthic', 'icevvelo', 'iceuvelo', 'sorunoff']
variables = ['kd']
in_dir = '/scratch/project_2001635/siiriasi/smartsea_data/syke_test/'
for var in variables:
for stat in stats:
out_dir = '/scratch/project_2001635/siiriasi/smartsea_data/syke_test/{}/'.format(stat)
the_command = "zip -v {}/{}_monthly_{}_seriesB.zip {}".format(\
out_dir, var, \
stat, \
" {}/{}*.nc".format(out_dir,var))
print(the_command)
# processes.add(the_command)
processes.wait_all()
print("All done!")