Data model for explicit storage of restricted runs and their courses

.annotation_safe_list.yml

@@ -21,6 +21,8 @@ catalog.HistoricalContentMetadata:
     ".. no_pii": "This model has no PII"
 catalog.HistoricalEnterpriseCatalog:
   ".. no_pii": "This model has no PII"
+catalog.HistoricalRestrictedCourseMetadata:
+    ".. no_pii": "This model has no PII"
 contenttypes.ContentType:
   ".. no_pii:": "This model has no PII"
 curation.HistoricalEnterpriseCurationConfig:

enterprise_catalog/apps/api/tasks.py

@@ -287,7 +287,7 @@ def _update_full_content_metadata_course(content_keys, dry_run=False):
         else:
             ContentMetadata.objects.bulk_update(
                 modified_content_metadata_records,
-                ['json_metadata'],
+                ['_json_metadata'],
                 batch_size=10,
             )
 
@@ -326,13 +326,14 @@ def _update_single_full_course_record(course_metadata_dict, metadata_record, cou
     for that course.
     """
     # Merge the full metadata from discovery's /api/v1/courses into the local metadata object.
-    metadata_record.json_metadata.update(course_metadata_dict)
+    metadata_record._json_metadata.update(course_metadata_dict)  # pylint: disable=protected-access
 
     _normalize_metadata_record(metadata_record)
 
     if course_review:
-        metadata_record.json_metadata['reviews_count'] = course_review.get('reviews_count')
-        metadata_record.json_metadata['avg_course_rating'] = course_review.get('avg_course_rating')
+        # pylint: disable=protected-access
+        metadata_record._json_metadata['reviews_count'] = course_review.get('reviews_count')
+        metadata_record._json_metadata['avg_course_rating'] = course_review.get('avg_course_rating')
 
     if metadata_record.json_metadata.get(FORCE_INCLUSION_METADATA_TAG_KEY):
         metadata_record.json_metadata = transform_course_metadata_to_visible(metadata_record.json_metadata)
@@ -358,11 +359,12 @@ def _normalize_metadata_record(course_metadata_record):
     normalized_metadata_input = {
         'course_metadata': course_metadata_record.json_metadata,
     }
-    course_metadata_record.json_metadata['normalized_metadata'] =\
+    # pylint: disable=protected-access
+    course_metadata_record._json_metadata['normalized_metadata'] =\
         NormalizedContentMetadataSerializer(normalized_metadata_input).data
-    course_metadata_record.json_metadata['normalized_metadata_by_run'] = {}
+    course_metadata_record._json_metadata['normalized_metadata_by_run'] = {}
     for run in course_metadata_record.json_metadata.get('course_runs', []):
-        course_metadata_record.json_metadata['normalized_metadata_by_run'].update({
+        course_metadata_record._json_metadata['normalized_metadata_by_run'].update({
             run['key']: NormalizedContentMetadataSerializer({
                 'course_run_metadata': run,
                 'course_metadata': course_metadata_record.json_metadata,
@@ -413,15 +415,15 @@ def _update_full_content_metadata_program(content_keys, dry_run=False):
                 logger.error('Could not find ContentMetadata record for content_key %s.', content_key)
                 continue
 
-            metadata_record.json_metadata.update(program_metadata_dict)
+            metadata_record._json_metadata.update(program_metadata_dict)  # pylint: disable=protected-access
             modified_content_metadata_records.append(metadata_record)
 
         if dry_run:
             logger.info('dry_run=true, not updating program metadata')
         else:
             ContentMetadata.objects.bulk_update(
                 modified_content_metadata_records,
-                ['json_metadata'],
+                ['_json_metadata'],
                 batch_size=10,
             )
 
@@ -1200,7 +1202,12 @@ def fetch_missing_course_metadata_task(self, force=False, dry_run=False):  # pyl
     that are embedded inside a program.
     """
     logger.info('[FETCH_MISSING_METADATA] fetch_missing_course_metadata_task task started.')
-    program_metadata_list = ContentMetadata.objects.filter(content_type=PROGRAM).values_list('json_metadata', flat=True)
+    program_metadata_list = ContentMetadata.objects.filter(
+        content_type=PROGRAM
+    ).values_list(
+        '_json_metadata',
+        flat=True,
+    )
     course_keys = set()
     for program_metadata in program_metadata_list:
         if program_metadata is not None:
@@ -1266,7 +1273,7 @@ def fetch_missing_pathway_metadata_task(self, force=False, dry_run=False):  # py
     )
 
     learner_pathway_metadata_list = ContentMetadata.objects.filter(content_type=LEARNER_PATHWAY).values_list(
-        'json_metadata', flat=True,
+        '_json_metadata', flat=True,
     )
     program_uuids = set()
     course_keys = set()

enterprise_catalog/apps/api/tests/test_tasks.py

@@ -223,7 +223,7 @@ def test_fetch_missing_course_metadata_task(self, mock_update_data_from_discover
         """
         test_course = 'course:edX+testX'
         course_content_metadata = ContentMetadataFactory.create(content_type=COURSE)
-        ContentMetadataFactory.create(content_type=PROGRAM, json_metadata={
+        ContentMetadataFactory.create(content_type=PROGRAM, _json_metadata={
             'courses': [
                 course_content_metadata.json_metadata,
                 {
@@ -489,7 +489,7 @@ def test_update_full_metadata(self, mock_oauth_client, mock_partition_course_key
         # the normalized metadata serializer.
         metadata_4 = ContentMetadataFactory(content_type=COURSE, content_key=course_key_4)
         metadata_4.catalog_queries.set([self.catalog_query])
-        metadata_4.json_metadata['advertised_course_run_uuid'] = None
+        metadata_4._json_metadata['advertised_course_run_uuid'] = None  # pylint: disable=protected-access
         metadata_4.save()
         non_course_metadata = ContentMetadataFactory(content_type=COURSE_RUN, content_key=non_course_key)
         non_course_metadata.catalog_queries.set([self.catalog_query])
@@ -714,7 +714,7 @@ def test_update_full_metadata_exec_ed(self, mock_oauth_client, mock_partition_co
 
         # Simulate a pre-existing ContentMetadata object freshly seeded using the response from /api/v1/search/all/
         course_metadata = ContentMetadataFactory.create(
-            content_type=COURSE, content_key=course_key, json_metadata={
+            content_type=COURSE, content_key=course_key, _json_metadata={
                 'aggregation_key': 'course:edX+testX',
                 'key': 'edX+testX',
                 'course_type': EXEC_ED_2U_COURSE_TYPE,
@@ -801,7 +801,8 @@ def setUp(self):
         self.course_metadata_published.catalog_queries.set([self.enterprise_catalog_query])
         self.course_metadata_published.tags.set([self.tag1])
         self.course_metadata_unpublished = ContentMetadataFactory(content_type=COURSE, content_key='course-2')
-        self.course_metadata_unpublished.json_metadata.get('course_runs')[0].update({
+        # pylint: disable=protected-access
+        self.course_metadata_unpublished._json_metadata.get('course_runs')[0].update({
             'status': 'unpublished',
         })
         self.course_metadata_unpublished.catalog_queries.set([self.enterprise_catalog_query])
@@ -821,7 +822,7 @@ def setUp(self):
             content_type=COURSE_RUN,
             parent_content_key='course-2',
         )
-        self.course_run_metadata_unpublished.json_metadata.update({
+        self.course_run_metadata_unpublished._json_metadata.update({
             'status': 'unpublished',
         })
         self.course_run_metadata_unpublished.catalog_queries.set([course_run_catalog_query])
@@ -874,7 +875,7 @@ def test_add_metadata_to_algolia_objects_skips_metadata_records_over_max_size(se
         short_description_string = "ayylmao".join(["" for x in range(50000)])
         full_description_string = "foobar".join(["" for x in range(50000)])
         too_big_sized_course = ContentMetadataFactory(content_type=COURSE, content_key='test-course-2')
-        too_big_sized_course.json_metadata.update(
+        too_big_sized_course._json_metadata.update(  # pylint: disable=protected-access
             {"short_description": short_description_string, "full_description": full_description_string}
         )
         too_big_sized_course.save()
@@ -1216,7 +1217,7 @@ def test_index_algolia_published_course_to_program(self, mock_search_client):
         program_2 = ContentMetadataFactory(content_type=PROGRAM, content_key='program-2')
 
         # Make program-2 hidden to make it "non-indexable". Later we will assert that it will not get indexed.
-        program_2.json_metadata.update({
+        program_2._json_metadata.update({  # pylint: disable=protected-access
             'hidden': True,
         })
         program_2.save()
@@ -1324,7 +1325,7 @@ def test_index_algolia_unpublished_course_to_program(self, mock_search_client):
         program_2.catalog_queries.set([self.enterprise_catalog_query])
 
         # Make program-2 hidden to make it "non-indexable". Later we will assert that it will not get indexed.
-        program_2.json_metadata.update({
+        program_2._json_metadata.update({  # pylint: disable=protected-access
             'hidden': True,
         })
         program_2.save()
@@ -1635,7 +1636,7 @@ def test_index_algolia_program_to_pathway(self, mock_search_client):
         program_1.catalog_queries.set([self.enterprise_catalog_query])
 
         # Make program-2 hidden to make it "non-indexable". Later we will assert that it will not get indexed.
-        program_2.json_metadata.update({
+        program_2._json_metadata.update({  # pylint: disable=protected-access
             'hidden': True,
         })
         program_2.save()
@@ -1772,15 +1773,15 @@ def test_index_algolia_all_uuids(self, mock_search_client):
         program_for_main_course = ContentMetadataFactory(content_type=PROGRAM, content_key='program-1')
         # Make the program hidden to make it "non-indexable", but ensure that it still gets indexed due to being related
         # to an indexable course.
-        program_for_main_course.json_metadata.update({
+        program_for_main_course._json_metadata.update({  # pylint: disable=protected-access
             'hidden': True,
         })
         program_for_main_course.save()
         program_for_pathway = ContentMetadataFactory(content_type=PROGRAM, content_key='program-2')
         program_for_pathway.catalog_queries.set([self.enterprise_catalog_query])
         # Make the program hidden to make it "non-indexable", but ensure that it still gets indexed due to being related
         # to an indexable pathway.
-        program_for_pathway.json_metadata.update({
+        program_for_pathway._json_metadata.update({  # pylint: disable=protected-access
             'hidden': True,
         })
         program_for_pathway.save()

enterprise_catalog/apps/api/v1/tests/test_serializers.py

@@ -29,9 +29,7 @@ def test_product_source_formatting(self):
         Test that the content metadata serializer will transform product source data within the json metadata field
         from a string to a dict.
         """
-        json_metadata = self.content_metadata_item.json_metadata
-        json_metadata['product_source'] = '2u'
-        self.content_metadata_item.json_metadata = json_metadata
+        self.content_metadata_item._json_metadata.update({'product_source': '2u'})  # pylint: disable=protected-access
         self.content_metadata_item.save()
         serialized_data = ContentMetadataSerializer(self.content_metadata_item)
         assert serialized_data.data.get('product_source') == {

enterprise_catalog/apps/api/v1/tests/test_utils.py

@@ -69,10 +69,10 @@ def test_get_archived_content_count(self):
         """
         Test that archived content will increment the count.
         """
-        content_1 = ContentMetadataFactory.create(json_metadata={'course_run_statuses': ['archived']})
-        content_2 = ContentMetadataFactory.create(json_metadata={'course_run_statuses': ['unpublished', 'archived']})
+        content_1 = ContentMetadataFactory.create(_json_metadata={'course_run_statuses': ['archived']})
+        content_2 = ContentMetadataFactory.create(_json_metadata={'course_run_statuses': ['unpublished', 'archived']})
         # if there's at least one published course run, the content should not be considered archived
-        content_3 = ContentMetadataFactory.create(json_metadata={'course_run_statuses': ['published', 'archived']})
+        content_3 = ContentMetadataFactory.create(_json_metadata={'course_run_statuses': ['published', 'archived']})
 
         highlighted_content_1 = HighlightedContentFactory(content_metadata=content_1)
         highlighted_content_2 = HighlightedContentFactory(content_metadata=content_2)

enterprise_catalog/apps/catalog/management/commands/tests/test_set_global_average_course_rating_value.py

@@ -25,13 +25,13 @@ def test_command_averages_course_reviews(
         """
         ContentMetadataFactory(
             content_type='course',
-            json_metadata={'avg_course_rating': 5, 'reviews_count': 20}
+            _json_metadata={'avg_course_rating': 5, 'reviews_count': 20}
         )
         ContentMetadataFactory(
             content_type='course',
-            json_metadata={'avg_course_rating': 4, 'reviews_count': 10}
+            _json_metadata={'avg_course_rating': 4, 'reviews_count': 10}
         )
-        ContentMetadataFactory(json_metadata={})
+        ContentMetadataFactory(_json_metadata={})
         call_command(self.command_name)
         expected_total_average = ((5 * 20) + (4 * 10)) / 30