-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
848a5ad
commit 87cdbc7
Showing
9 changed files
with
156 additions
and
97 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,21 +1,37 @@ | ||
FROM ubuntu:22.04 | ||
|
||
USER root | ||
ARG DEBIAN_FRONTEND=noninteractive | ||
RUN apt update -y | ||
RUN apt install -y software-properties-common | ||
RUN add-apt-repository ppa:deadsnakes/ppa && apt update -y | ||
|
||
# Install Python and Pip | ||
RUN apt install -y python3-pip && pip install psycopg2-binary | ||
# Install Python3.12 and Pip | ||
RUN apt install -y python3.12 python3-pip | ||
|
||
# Install Ansible dependencies | ||
RUN apt install -y git python3.12-venv python3.12-dev | ||
|
||
# Install psycopg2-binary | ||
RUN pip install psycopg2-binary | ||
RUN pip3 install psycopg2-binary | ||
|
||
# Install Ansible | ||
RUN apt install -y ansible | ||
|
||
# Create task_log file | ||
RUN touch /task_log | ||
# Copy gTS build resources to the container | ||
COPY ./build-resources/geocml-task-scheduler/ /geocml-task-scheduler | ||
|
||
# Copy gTS to the container | ||
COPY ./build-resources/geocml-task-scheduler/geocml-task-scheduler/ /geocml-task-scheduler | ||
# Install Ansible dependencies and run through playbook | ||
COPY ./ansible-playbooks/geocml-task-scheduler-requirements.yaml ./ansible-playbooks/geocml-task-scheduler-playbook.yaml ./ | ||
RUN ansible-galaxy collection install ansible.posix && ansible-galaxy install -r geocml-task-scheduler-requirements.yaml && ansible-playbook -i,localhost geocml-task-scheduler-playbook.yaml --tags "all" && rm -f ./*.yaml | ||
|
||
######### Customize Container Here ########### | ||
######### End Customizations ########### | ||
|
||
CMD python3 /geocml-task-scheduler/schedule.py | ||
# Uninstall Ansible stuff | ||
RUN rm -rf $HOME/.ansible && apt purge -y ansible* && apt purge -y git* && apt purge -y virtualenv* | ||
|
||
# Remove install cache | ||
RUN apt clean autoclean && apt autoremove -y && rm -rf /var/lib/{apt,dpkg,cache,log}/ | ||
|
||
CMD python3 /geocml-task-scheduler/geocml-task-scheduler/schedule.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
--- | ||
- # Configure geocml-task-scheduler | ||
hosts: localhost | ||
connection: local | ||
gather_facts: yes | ||
become: yes | ||
tasks: | ||
- name: Create task_log file | ||
ansible.builtin.command: touch /task_log | ||
- name: Clone Tabor source | ||
ansible.builtin.command: | ||
chdir: /geocml-task-scheduler | ||
cmd: git clone https://github.com/geoCML/tabor.git | ||
- name: Update Tabor submodule to latest release tag | ||
shell: | | ||
cd /geocml-task-scheduler/tabor | ||
git fetch --tags | ||
TAG=$(git tag | tail -1) | ||
#git checkout $TAG | ||
- name: Create python venv for Tabor build | ||
ansible.builtin.command: | ||
chdir: /geocml-task-scheduler/tabor | ||
cmd: python3.12 -m venv ./venv | ||
- name: Install Tabor requirements | ||
ansible.builtin.pip: | ||
chdir: /geocml-task-scheduler/tabor | ||
virtualenv: ./venv | ||
requirements: ./requirements.txt | ||
- name: Build Tabor | ||
shell: | | ||
cd /geocml-task-scheduler/tabor | ||
source ./venv/bin/activate && python3.12 -m pip install -U pyinstaller==6.9.0 && pyinstaller --paths=./src -y ./src/tabor.py | ||
args: | ||
executable: /bin/bash |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
--- | ||
# Add Ansible Galaxy Packages here, role_example_hello included to prevent errors in template testing | ||
|
||
# NOTE: Uncomment the two lines below if you want to test your Ansible installation | ||
# before proceeding with the rest of the playbook. | ||
# | ||
# This is typically only necessary if you are changing the default Ansible installation | ||
# that comes with the geoCML base image. | ||
#roles: | ||
#- irixjp.role_example_hello # https://galaxy.ansible.com/irixjp/role_example_hello | ||
collections: | ||
- community.general |
95 changes: 37 additions & 58 deletions
95
build-resources/geocml-task-scheduler/geocml-task-scheduler/backup_geocml_db.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,88 +1,67 @@ | ||
import psycopg2 | ||
import os | ||
import subprocess | ||
from time import time | ||
from task_logger import log | ||
|
||
ignore_tables = ('spatial_ref_sys', 'geometry_columns', 'geography_columns') | ||
ignore_schemas = ('pg_catalog', 'information_schema') | ||
ignore_tables = ("spatial_ref_sys", "geometry_columns", "geography_columns") | ||
ignore_schemas = ("pg_catalog", "information_schema") | ||
|
||
def backup_geocml_db(): | ||
try: | ||
conn = psycopg2.connect(dbname='geocml_db', | ||
user='postgres', | ||
password='admin', | ||
host='geocml-postgres', | ||
conn = psycopg2.connect(dbname="geocml_db", | ||
user="geocml", | ||
password="geocml", | ||
host="geocml-postgres", | ||
port=5432) | ||
except psycopg2.OperationalError: | ||
log('Couldn\'t connect to geocml_db; is the postgresql service started?') | ||
log("Couldn\'t connect to geocml_db; is the postgresql service started?") | ||
return | ||
|
||
cursor = conn.cursor() | ||
cursor.execute('SELECT DISTINCT table_schema FROM information_schema.tables;') | ||
schemas = cursor.fetchall() | ||
back_up_timestamp = time() | ||
path_to_backup_dir = os.path.join(os.sep, 'DBBackups', str(back_up_timestamp)) | ||
path_to_backup_dir = os.path.join(os.sep, "DBBackups", str(back_up_timestamp)) | ||
os.mkdir(path_to_backup_dir) | ||
delete_backup_dir = True | ||
|
||
|
||
# Write table schemata to .tabor file | ||
out = subprocess.run(["tabor", "write", "--db", "geocml_db", | ||
"--username", "postgres", "--password", "admin", | ||
"--host", "geocml-postgres", | ||
"--file", os.path.join(path_to_backup_dir, "geocml_db.tabor")], | ||
capture_output=True) | ||
|
||
if out.stderr: | ||
log("Failed to generate .tabor file {}".format(out.stderr)) | ||
os.rmdir(path_to_backup_dir) | ||
return | ||
|
||
cursor = conn.cursor() | ||
cursor.execute("""SELECT DISTINCT table_schema FROM information_schema.tables;""") | ||
schemas = cursor.fetchall() | ||
|
||
# Write table data to CSV file | ||
for schema in schemas: | ||
if schema[0] in ignore_schemas: | ||
continue | ||
cursor.execute('SELECT * FROM information_schema.tables WHERE table_schema = \'{}\';' | ||
.format(schema[0])) | ||
tables = cursor.fetchall() | ||
|
||
cursor.execute(f"""SELECT * FROM information_schema.tables WHERE table_schema = '{schema[0]}';""") | ||
|
||
tables = cursor.fetchall() | ||
|
||
for table in tables: | ||
if table[2] in ignore_tables: | ||
continue | ||
|
||
delete_backup_dir = False | ||
|
||
# Write to schema file | ||
schema_file_path = os.path.join(path_to_backup_dir, 'schema:{}.{}.sql'.format(schema[0], table[2])) | ||
schema_file = open(schema_file_path, 'w') | ||
|
||
if not schema[0] == 'public': | ||
cursor.execute('SELECT DISTINCT grantee FROM information_schema.role_table_grants WHERE table_schema = \'{}\';' | ||
.format(schema[0])) | ||
schema_owner = cursor.fetchall() | ||
schema_file.write('CREATE SCHEMA IF NOT EXISTS {} AUTHORIZATION {};\n' | ||
.format(schema[0], schema_owner[0][0])) | ||
|
||
cursor.execute('SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE contype = \'p\' AND conrelid::regclass::text LIKE \'%{}%\';'.format(table[2])) | ||
|
||
pk = cursor.fetchall() | ||
|
||
cursor.execute('SELECT column_name, udt_name FROM information_schema.columns WHERE table_name = \'{}\';' | ||
.format(table[2])) | ||
|
||
columns_and_datatypes = [] | ||
for row in cursor: | ||
if len(row) == 3: # column has a constraint | ||
columns_and_datatypes.append('{} {} {}'.format(row[0], row[1], row[2])) | ||
else: | ||
columns_and_datatypes.append('{} {}'.format(row[0], row[1])) | ||
columns_and_datatypes = ', '.join(columns_and_datatypes) | ||
|
||
if len(pk) > 0: # table has primary key (expected) | ||
schema_file.write('CREATE TABLE IF NOT EXISTS {}."{}" ({}, {});\n'.format(schema[0], table[2], columns_and_datatypes, pk[0][0])) | ||
else: | ||
schema_file.write('CREATE TABLE IF NOT EXISTS {}."{}" ({});\n'.format(schema[0], table[2], columns_and_datatypes)) | ||
|
||
cursor.execute('SELECT tableowner FROM pg_tables WHERE tablename = \'{}\';'.format(table[2])) | ||
table_owner = cursor.fetchall() | ||
|
||
schema_file.write('ALTER TABLE {}."{}" OWNER TO {};'.format(schema[0], table[2], table_owner[0][0])) | ||
schema_file.close() | ||
|
||
# Write to data file | ||
data_file_path = os.path.join(path_to_backup_dir, 'data:{}.{}.csv'.format(schema[0], table[2])) | ||
data_file = open(data_file_path, 'w') | ||
cursor.copy_expert('COPY {}."{}" TO STDOUT WITH (FORMAT csv, DELIMITER \',\', HEADER FALSE);'.format(schema[0], table[2]), data_file) | ||
data_file_path = os.path.join(path_to_backup_dir, "data:{}.{}.csv".format(schema[0], table[2])) | ||
data_file = open(data_file_path, "w") | ||
cursor.copy_expert(f"""COPY {schema[0]}."{table[2]}" TO STDOUT WITH (FORMAT csv, DELIMITER ',', HEADER);""", data_file) | ||
data_file.close() | ||
|
||
if delete_backup_dir: # nothing to back up | ||
path_to_backup_dir.rmdir() | ||
log("Nothing to backup") | ||
os.rmdir(path_to_backup_dir) | ||
|
||
cursor.close() | ||
conn.close() |
69 changes: 43 additions & 26 deletions
69
...d-resources/geocml-task-scheduler/geocml-task-scheduler/restore_geocml_db_from_backups.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters