Feature/batch program

compass/dao/contact.py

@@ -43,10 +43,6 @@ def validate_student_systemkey(student_systemkey):
         raise ValueError(f"Invalid student systemkey: {e}")
 
 
-def pad_student_systemkey(student_systemkey):
-    return student_systemkey.zfill(9)
-
-
 def parse_checkin_date_str(checkin_date_str):
     # parse checkin date
     if checkin_date_str is None:

compass/dao/csv.py

@@ -0,0 +1,103 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+from compass.dao.person import (
+    get_person_by_student_number, PersonNotFoundException)
+from compass.utils import format_student_number
+from compass.exceptions import InvalidCSV
+from logging import getLogger
+import chardet
+import csv
+import os
+
+logger = getLogger(__name__)
+
+
+def normalize(s):
+    return s.replace(' ', '').replace('_', '').lower()
+
+
+class InsensitiveDict(dict):
+    """
+    Override get() to strip() and lower() the input key, and strip() the
+    returned value.
+    """
+    def get(self, *k, default=None):
+        for i in k:
+            if normalize(i) in self:
+                try:
+                    return super().get(normalize(i)).strip()
+                except AttributeError:
+                    break
+        return default
+
+
+class InsensitiveDictReader(csv.DictReader):
+    """
+    Override the csv.fieldnames property to strip() and lower() the fieldnames.
+    """
+    @property
+    def fieldnames(self):
+        return [normalize(field) for field in super().fieldnames]
+
+    def __next__(self):
+        return InsensitiveDict(super().__next__())
+
+
+class StudentCSV():
+    def __init__(self):
+        self.encoding = None
+        self.student_identifiers = [
+            'studentid', 'studentno', 'studentnum', 'studentnumber',
+            'studentidnumber']
+
+    def decode_file(self, csvfile):
+        if not self.encoding:
+            result = chardet.detect(csvfile)
+            self.encoding = result['encoding']
+        return csvfile.decode(self.encoding)
+
+    def validate(self, fileobj):
+        # Read the first line of the file to validate the header
+        decoded_file = self.decode_file(fileobj.readline())
+        self.has_header = csv.Sniffer().has_header(decoded_file)
+        self.dialect = csv.Sniffer().sniff(decoded_file)
+
+        reader = InsensitiveDictReader(decoded_file.splitlines(),
+                                       dialect=self.dialect)
+
+        if not (any(s in reader.fieldnames for s in self.student_identifiers)):
+            raise InvalidCSV('Missing student identifier')
+
+        fileobj.seek(0, 0)
+
+    def persons_from_file(self, fileobj):
+        """
+        Reads a CSV file object, and returns a list of person JSON objects
+
+        Supported column names are contained in self.student_identifiers
+
+        All other field names are ignored.
+        """
+        self.validate(fileobj)
+        decoded_file = self.decode_file(fileobj.read()).splitlines()
+
+        persons = []
+        for row in InsensitiveDictReader(decoded_file, dialect=self.dialect):
+            student_number = format_student_number(
+                row.get(*self.student_identifiers))
+
+            if student_number is None:
+                persons.append({'error': 'Missing student number'})
+                continue
+
+            try:
+                person = get_person_by_student_number(student_number)
+                person_dict = person.to_dict()
+                person_dict['student_number'] = student_number
+                persons.append(person_dict)
+            except PersonNotFoundException:
+                persons.append({
+                    'error': f'Student number {student_number} not found'})
+
+        return persons

compass/exceptions.py

@@ -2,16 +2,19 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-"""
-Custom exceptions used by Compass.
-"""
-
-
 class OverrideNotPermitted(Exception):
     def __str__(self):
         return "Action not permitted while using admin override"
 
 
 class InvalidSystemKey(Exception):
     def __str__(self):
-        return "system_ey is invalid"
+        return "system_key is invalid"
+
+
+class InvalidCSV(Exception):
+    def __init__(self, error="Invalid CSV file"):
+        self.error = error
+
+    def __str__(self):
+        return self.error

compass/fixtures/uw_person/student.json

@@ -47,7 +47,7 @@
             "ethnic_group_desc": "White",
             "exemption_code": "0",
             "exemption_desc": "NONE",
-            "external_email": "javerage@gmail.com",
+            "external_email": "lisa@test.com",
             "first_generation_4yr_ind": false,
             "first_generation_ind": false,
             "gender": "F",
@@ -111,7 +111,7 @@
             "spp_category": "1",
             "spp_category_dt": "2022-07-01 04:00:09.613-08:00",
             "sr_col_gpa": "0.00",
-            "student_email": "javerage@uw.edu",
+            "student_email": "lisa@uw.edu",
             "student_number": "1233338",
             "system_key": "888777333",
             "total_credits": "79.00",

compass/management/commands/process_omad_contacts.py

@@ -2,15 +2,15 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from django.core.management.base import BaseCommand
-from compass.models import OMADContactQueue
+from compass.models import (
+    AccessGroup, AppUser, Contact, Student, OMADContactQueue)
+from compass.dao.contact import validate_contact_post_data
+from compass.utils import format_system_key
 from datetime import datetime, timezone
 import json
-from compass.models import (
-    AccessGroup, AppUser, Contact, Student)
 from logging import getLogger
-from compass.dao.contact import (validate_contact_post_data,
-                                 pad_student_systemkey)
 import traceback
+
 logger = getLogger(__name__)
 
 
@@ -52,7 +52,7 @@ def process_contact(contact):
             contact_dict["adviser_netid"])
 
         # Parse/format data
-        student_systemkey = pad_student_systemkey(
+        student_systemkey = format_system_key(
             contact_dict["student_systemkey"])
 
         student, _ = Student.objects.get_or_create(

compass/resources/csv/insensitive.csv

@@ -0,0 +1,2 @@
+"field1","  field2","Field3",FIELD4,"field5","field 6","field7",Field_8
+"ök1",øk2,ok3,ok4,"   ok5  ","   ",,ok_8

compass/resources/csv/missing_header.csv

@@ -0,0 +1,6 @@
+"800000","javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+"40000","jjulius","Julius","J","AB","2.0","GRADUATE","Oceanography","[email protected]"
+"1000000","baverage","Average","Bill P","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"7000000","student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"9000000","student2","Student","2","AA","2.0","NON_MATRIC","Non Matriculated","[email protected]"
+"2000000","student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/resources/csv/missing_student_id.csv

@@ -0,0 +1,7 @@
+UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email
+"javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+"jjulius","Julius","J","AB","2.0","GRADUATE","Oceanography","[email protected]"
+"baverage","Average","Bill P","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"student2","Student","2","AA","2.0","NON_MATRIC","Non Matriculated","[email protected]"
+"student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/resources/csv/valid.csv

@@ -0,0 +1,6 @@
+Student_No,UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email
+"1033334","javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+"1233338","lisa","Simpson","Lisa","AB","2.0","GRADUATE","Oceanography","[email protected]"
+"1233334","jbothell","Bothell","J","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"7000000","student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"","student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/tests/dao/test_csv.py

@@ -0,0 +1,65 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+from compass.dao.csv import InsensitiveDictReader, StudentCSV
+from compass.exceptions import InvalidCSV
+from compass.tests import CompassTestCase
+import mock
+import os
+
+
+class StudentCSVTest(CompassTestCase):
+    def setUp(self):
+        self.resource_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), '..', '..', 'resources', 'csv'))
+
+    def test_validate(self):
+        csv_import = StudentCSV()
+
+        with open(os.path.join(self.resource_path, 'valid.csv'), 'rb') as fh:
+            r = csv_import.validate(fh)
+            self.assertEqual(csv_import.has_header, True)
+            self.assertEqual(csv_import.dialect.delimiter, ',')
+
+        with open(os.path.join(
+                self.resource_path, 'missing_header.csv'), 'rb') as fh:
+            self.assertRaisesRegex(InvalidCSV, 'Missing student identifier',
+                                   csv_import.validate, fh)
+
+        with open(os.path.join(
+                self.resource_path, 'missing_student_id.csv'), 'rb') as fh:
+            self.assertRaisesRegex(InvalidCSV, 'Missing student identifier',
+                                   csv_import.validate, fh)
+
+    def test_persons_from_file(self):
+        csv_import = StudentCSV()
+
+        with open(os.path.join(self.resource_path, 'valid.csv'), 'rb') as fh:
+            result = csv_import.persons_from_file(fh)
+
+            self.assertEqual(len(result), 5)
+            self.assertEqual(result[0]['student_number'], '1033334')
+            self.assertEqual(result[0]['uwnetid'], 'javerage')
+            self.assertEqual(result[1]['student_number'], '1233338')
+            self.assertEqual(result[1]['uwnetid'], 'lisa')
+            self.assertEqual(result[2]['student_number'], '1233334')
+            self.assertEqual(result[2]['uwnetid'], 'jbothell')
+            self.assertEqual(result[3]['error'],
+                             'Student number 7000000 not found')
+            self.assertEqual(result[4]['error'], 'Missing student number')
+
+
+class InsensitiveDictReaderTest(StudentCSVTest):
+    def test_insensitive_dict_reader(self):
+        with open(os.path.join(self.resource_path, 'insensitive.csv')) as fh:
+            reader = InsensitiveDictReader(fh)
+
+            row = next(reader)
+            self.assertEqual(row.get('Field1'), 'ök1')
+            self.assertEqual(row.get('Field2'), 'øk2')
+            self.assertEqual(row.get('Field3'), 'ok3')
+            self.assertEqual(row.get('Field4'), 'ok4')
+            self.assertEqual(row.get('Field 5', 'Field5'), 'ok5')
+            self.assertEqual(row.get('Field6', 'field 6'), '')
+            self.assertEqual(row.get('Field7'), '')
+            self.assertEqual(row.get('Field8'), 'ok_8')

compass/tests/test_utils.py

@@ -2,11 +2,38 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from django.test import TestCase
-from compass.utils import weekdays_before
+from compass.utils import (
+    weekdays_before, format_system_key, format_student_number)
 from datetime import datetime, date
 
 
 class TestUtils(TestCase):
+    def test_format_system_key(self):
+        self.assertEqual(format_system_key('123'), '000000123')
+        self.assertEqual(format_system_key('1234567'), '001234567')
+        self.assertEqual(format_system_key('12345678'), '012345678')
+        self.assertEqual(format_system_key('123456789'), '123456789')
+        self.assertEqual(format_system_key('1234567890'), '1234567890')
+        self.assertEqual(format_system_key(1234), '000001234', 'integer')
+        self.assertEqual(format_system_key('1abcdef'), None)
+        self.assertEqual(format_system_key(1.55), None, 'decimal')
+        self.assertEqual(format_system_key(0), None, 'zero')
+        self.assertEqual(format_system_key('000'), None)
+        self.assertEqual(format_system_key(None), None)
+        self.assertEqual(format_system_key(''), None)
+
+    def test_format_student_number(self):
+        self.assertEqual(format_student_number('123'), '0000123')
+        self.assertEqual(format_student_number('123456'), '0123456')
+        self.assertEqual(format_student_number('1234567890'), '1234567890')
+        self.assertEqual(format_student_number(1234), '0001234', 'integer')
+        self.assertEqual(format_student_number('1abcdef'), None)
+        self.assertEqual(format_student_number(1.55), None, 'decimal')
+        self.assertEqual(format_student_number(0), None, 'zero')
+        self.assertEqual(format_student_number('000'), None)
+        self.assertEqual(format_student_number(None), None)
+        self.assertEqual(format_student_number(''), None)
+
     def test_weekdays_before(self):
         # date
         val = weekdays_before(date(2024, 4, 19), offset_days=3)

compass/urls.py

@@ -13,6 +13,7 @@
     StudentSchedulesView,
     StudentTranscriptsView,
     StudentAffiliationsView,
+    StudentAffiliationsImportView,
     StudentVisitsView,
     StudentEligibilityView,
     StudentView,
@@ -87,6 +88,10 @@
         r"^api/internal/student/(?P<systemkey>[\w]+)/affiliations/$",
         StudentAffiliationsView.as_view(),
     ),
+    re_path(
+        r"^api/internal/student/affiliations/(?P<affiliation_id>[\w]+)/import/$",  # noqa
+        StudentAffiliationsImportView.as_view(),
+    ),
     re_path(
         r"^api/internal/student/(?P<systemkey>[\w]+)"
         r"/affiliations/(?P<affiliation_id>[\w]+)/$",
@@ -170,7 +175,10 @@
         name="visit_omad"
     ),
     # vue-router paths
-    re_path(r"^(checkins|student|caseload|reports).*$", LandingView.as_view()),
+    re_path(
+        r"^(checkins|student|caseload|reports|affiliations).*$",
+        LandingView.as_view()
+    ),
     # default landing
     re_path(r"^$", LandingView.as_view(), name="index"),
 ]

compass/utils.py

@@ -3,6 +3,25 @@
 
 from datetime import timedelta
 
+STUDENT_NUM_LEN = 7
+STUDENT_SYSKEY_LEN = 9
+
+
+def zfill_or_none(value, length):
+    value = str(value).zfill(length) if (
+        value and str(value).isdigit()) else None
+    if value == '0' * length:
+        return None
+    return value
+
+
+def format_system_key(value):
+    return zfill_or_none(value, STUDENT_SYSKEY_LEN)
+
+
+def format_student_number(value):
+    return zfill_or_none(value, STUDENT_NUM_LEN)
+
 
 def weekdays_before(end_date, offset_days=3):
     """