diff --git a/.env b/.env new file mode 100644 index 0000000..bada14c --- /dev/null +++ b/.env @@ -0,0 +1,56 @@ +# DB image settings +POSTGRES_PASSWORD=ckan +DATASTORE_READONLY_PASSWORD=datastore + +# Basic +CKAN_SITE_ID=default +CKAN_SITE_URL=http://ckan.iaea.local:5000 +CKAN_PORT=5000 +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +TZ=UTC + +# Database connections (TODO: avoid duplication) +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@iaea_db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@iaea_db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@iaea_db/datastore + +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@iaea_db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@iaea_db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@iaea_db/datastore_test + +# Other services connections +CKAN_SOLR_URL=http://iaea_solr:8983/solr/ckan +CKAN_REDIS_URL=redis://iaea_redis:6379/1 +#CKAN_DATAPUSHER_URL=http://datapusher:8800 +#CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +TEST_CKAN_SOLR_URL=http://iaea_solr:8983/solr/ckan +TEST_CKAN_REDIS_URL=redis://iaea_redis:6379/1 + +# Core settings +CKAN__STORAGE_PATH=/var/lib/ckan + +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost + +## Extensions +#CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore datapusher iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations +CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore +#CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view geojson_view +CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=iaea_redis +CKAN__HARVEST__MQ__PORT=6379 +CKAN__HARVEST__MQ__REDIS_DB=1 + +# Sentry +# CKAN___SENTRY__DSN=https://xxxxxx:xxxxxx@sentry.domain.com/1 +# CKAN___SENTRY__CONFIGURE_LOGGING=True +# CKAN___SENTRY__LOG_LEVEL=WARN + diff --git a/ckanext/iaea/middleware.py b/ckanext/iaea/middleware.py new file mode 100644 index 0000000..279b175 --- /dev/null +++ b/ckanext/iaea/middleware.py @@ -0,0 +1,18 @@ +class RestrictMiddleware(object): + + def __init__(self, app, app_config): + self.app=app + + def __call__(self, environ, start_response): + ui_path = environ.get('PATH_INFO') + + if ui_path == "/stats" and not 'repoze.who.identity' in environ: + + status = u'404 Not Found' + location = u'/user/login' + headers = [(u'Location',location), (u'Content-type', u'text/plain')] + body='Not authorized to see this page' + start_response (status, headers) + return[body] + else: + return self.app(environ, start_response) \ No newline at end of file diff --git a/ckanext/iaea/plugin.py b/ckanext/iaea/plugin.py index 85ad1c5..3d13be2 100644 --- a/ckanext/iaea/plugin.py +++ b/ckanext/iaea/plugin.py @@ -1,10 +1,29 @@ import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit +import ckan.logic as logic from ckan.lib.plugins import DefaultTranslation -from ckanext.iaea.helpers import get_helpers from ckanext.iaea.logic import action, validators -import ckan.logic as logic +import ckanext.iaea.logic.auth as ia +from flask import Blueprint +from ckanext.iaea import view import ckan.model as model +import ckanext.iaea.middleware as middleware +from ckan.model.meta import engine +#from threading import Thread, Event +#import signal +import sys +from logging import getLogger +import os + +#import ckanext.iaea.profiler as profiler + +from ckanext.iaea.helpers import get_helpers + + +def package_activity_html(id): + activity = logic.get_action( + 'package_activity_list_html')({}, {'id': id ,'limit': 8}) + return activity def featured_group(): @@ -20,11 +39,79 @@ def featured_group(): return {} -class IaeaPlugin(plugins.SingletonPlugin, DefaultTranslation): - plugins.implements(plugins.IConfigurer) +def suggested_filter_fields_serializer(datapackage, view_dict): + suggested_filter_fields = view_dict.get('suggested_filter_fields', False) + try: + fields = datapackage['resources'][0]['schema']['fields'] + except KeyError as e: + fields = [] + rules = [] + date = {} + if suggested_filter_fields: + suggested_fields_with_type = [field for field in fields if field['name'] in suggested_filter_fields] + for field in suggested_fields_with_type: + if field['type'] in ['datetime', 'date']: + date = { + 'startDate': None, + 'endDate': None, + 'fieldName': field['name'] + } + else: + rules.append({ + 'combinator': 'AND', + 'field': field['name'], + 'operator': '=', + 'value': '' + }) + if rules: + datapackage['resources'][0].update({'rules': rules}) + if date: + datapackage['resources'][0].update({'date': date}) + return datapackage + + +def featured_view_url(pkg): + featured_view = model.ResourceView.get(pkg['featured_view']) + return toolkit.h.url_for(qualified=True, controller='dataset_resource', + action='view', id=pkg['name'], + resource_id=featured_view.resource_id, + view_id=featured_view.id) + + +class IaeaPlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm, + DefaultTranslation): plugins.implements(plugins.ITranslation) - plugins.implements(plugins.ITemplateHelpers, inherit=True) + plugins.implements(plugins.IConfigurer) + plugins.implements(plugins.IDatasetForm) + plugins.implements(plugins.IActions) plugins.implements(plugins.IValidators) + plugins.implements(plugins.IBlueprint) + plugins.implements(plugins.ITemplateHelpers, inherit=True) + plugins.implements(plugins.IMiddleware, inherit=True) + plugins.implements(plugins.IAuthFunctions) + + # IDatasetForm + def update_package_schema(self): + schema = super(IaeaPlugin, self).update_package_schema() + schema.update({ + u'featured_view': [toolkit.get_validator(u'ignore_missing'), + toolkit.get_converter(u'convert_to_extras')] + }) + return schema + + def show_package_schema(self): + schema = super(IaeaPlugin, self).show_package_schema() + schema.update({ + u'featured_view': [toolkit.get_converter(u'convert_from_extras'), + toolkit.get_validator(u'ignore_missing')], + }) + return schema + + def is_fallback(self): + return True + + def package_types(self): + return [] # IConfigurer @@ -38,15 +125,39 @@ def update_config(self, config_): def get_helpers(self): iaea_helpers = { 'featured_group': featured_group, - # 'package_activity_html': package_activity_html, - # 'suggested_filter_fields_serializer': suggested_filter_fields_serializer, - # 'featured_view_url': featured_view_url, + 'package_activity_html': package_activity_html, + 'suggested_filter_fields_serializer': suggested_filter_fields_serializer, + 'featured_view_url': featured_view_url, } iaea_helpers.update(get_helpers()) return iaea_helpers + # IActions + def get_actions(self): + return { + 'resource_view_create': action.resource_view_create, + 'resource_view_update': action.resource_view_update, + } + # IValidators def get_validators(self): return { 'iaea_owner_org_validator': validators.package_organization_validator, } + + # IBlueprint + def get_blueprint(self): + blueprint = Blueprint(self.name, self.__module__) + blueprint.template_folder = u'templates' + # Add plugin url rules to Blueprint object + blueprint.add_url_rule(u'/dataset/metadata/', view_func=view.metadata) + blueprint.add_url_rule(u'/dataset//view', view_func=view.FeatureView.as_view(str(u'feature_view'))) + return blueprint + + # IAuthFunctions + def get_auth_functions(self): + return {'package_create': ia.package_create} + + def make_middleware(self, app, config): + + return middleware.RestrictMiddleware(app, config) diff --git a/ckanext/iaea/templates/package/base.html b/ckanext/iaea/templates/package/base.html new file mode 100644 index 0000000..2443fc0 --- /dev/null +++ b/ckanext/iaea/templates/package/base.html @@ -0,0 +1,28 @@ +{% extends "page.html" %} + +{% set pkg = c.pkg_dict or pkg_dict %} +{% set current_url = h.full_current_url() %} + +{% block breadcrumb_content_selected %} class="active"{% endblock %} + +{% block subtitle %}{{ _('Datasets') }}{% endblock %} + +{% block breadcrumb_content %} +{% if pkg %} + {% set dataset = h.dataset_display_name(pkg) %} + {% if pkg.organization %} + {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %} + {% set group_type = pkg.organization.type %} +
  • {% link_for _('Organizations'), controller='organization', action='index', named_route=group_type + '_index' %}
    • +
  • {% link_for organization|truncate(30), controller='organization', action='read', id=pkg.organization.name, + named_route=group_type + '_read' %}
    • + {% else %} +
  • {% link_for _('Datasets'), controller='dataset', action='search' %}
    • +{% endif %} + {% link_for dataset|truncate(30), controller='dataset', action='read', + id=pkg.name %} +{% else %} +
  • {% link_for _('Datasets'), controller='dataset', action='search' %}
    • +
  • {{ _('Create Dataset') }}
    • +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/features_view.html b/ckanext/iaea/templates/package/features_view.html new file mode 100644 index 0000000..17f79b1 --- /dev/null +++ b/ckanext/iaea/templates/package/features_view.html @@ -0,0 +1,37 @@ +{% extends "package/read_base.html" %} + +{% block primary_content_inner %} +{% if package_views %} + +
    +
    +

    Select a view that should be featured on the dataset page.

    + {% for resource in package_views %} +

    {{ resource.resource_name }}

    + {% for view in resource.views %} +
    + {% set view_url = h.url_for(qualified=True, controller='dataset_resource', + action='view', id=pkg['name'], + resource_id=view.resource_id, + view_id=view.id) %} + {% if pkg['featured_view'] == view.id %} + + {{view.title}} + {% else %} + + {{view.title}} + {% endif %} +
    + {% endfor %} + {% endfor %} +
    + +
    + + +
    +
    +{% else %} +

    {{ _("There are no views created for this dataset yet.") }}

    +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/metadata.html b/ckanext/iaea/templates/package/metadata.html new file mode 100644 index 0000000..290289c --- /dev/null +++ b/ckanext/iaea/templates/package/metadata.html @@ -0,0 +1,6 @@ +{% extends "package/read_base.html" %} +{# Reuse of activity template for additonal info#} + +{% block primary_content_inner %} +{% snippet "package/snippets/additional_info.html", pkg_dict=pkg %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/read.html b/ckanext/iaea/templates/package/read.html new file mode 100644 index 0000000..c349485 --- /dev/null +++ b/ckanext/iaea/templates/package/read.html @@ -0,0 +1,11 @@ +{% ckan_extends %} + +{% block package_description %} +{% endblock %} + +{% block package_resources %} + {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources %} +{% endblock %} + +{% block package_additional_info %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/read_base.html b/ckanext/iaea/templates/package/read_base.html index d224d03..aee1a48 100644 --- a/ckanext/iaea/templates/package/read_base.html +++ b/ckanext/iaea/templates/package/read_base.html @@ -1,4 +1,160 @@ {% ckan_extends %} + +{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ super() }}{% endblock %} + +{% block head_extras -%} + {{ super() }} + {% set description = h.markdown_extract(pkg.notes, extract_length=200)|forceescape %} + + +{% endblock -%} + + + +{% block content_action %} + {% if h.check_access('package_update', {'id':pkg.id }) %} + {% link_for _('Manage'), controller='dataset', action='edit', id=pkg.name, class_='btn btn-default', icon='wrench' %} + {% endif %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }} + {{ h.build_nav_icon('iaea.metadata', _('Metadata'), id=pkg.name, icon='fa fa-tag') }} + {% if h.check_access('package_update', {'id':pkg.id }) %} + {{ h.build_nav_icon('iaea.feature_view', _('View'), id=pkg.name, icon='check-square-o') }} + {% endif %} +{% endblock %} + +{% block primary_content_inner %} + {% block package_revision_info %} + {% if c.revision_date %} +
    +

    + {% set timestamp = h.render_datetime(c.revision_date, with_hours=True) %} + {% set url = h.url_for(controller='dataset', action='read', id=pkg.name) %} + + {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the current revision.{% endtrans %} +

    +
    + {% endif %} + {% endblock %} +{% endblock %} +{% block pre_primary %} +
    +
    +
    + {% if pkg.private %} + + + {{ _('Private') }} + + {% endif %} +

    + {% block page_heading %} + {{ h.dataset_display_name(pkg) }} + {% if pkg.state.startswith('draft') %} + [{{ _('Draft') }}] + {% endif %} + {% if pkg.state == 'deleted' %} + [{{ _('Deleted') }}] + {% endif %} + {% endblock %} +

    + {% block package_notes %} + {% if pkg.notes %} +
    + {{ h.render_markdown(h.get_translated(pkg, 'notes')) }} +
    + {% endif %} + {% endblock %} +
    +
    +
    + {% block title_and_actions_right_side %} + {% if pkg.organization %} + {% set org = h.get_organization(pkg.organization.name) %} + {% set url_org = h.url_for(pkg.organization.type + '_read', id=pkg.organization.name, ) %} +
    +
    + + {{ pkg.organization.name }} + +
    +
    + {% endif %} + {% endblock %} +
    +
    +
    +
    +
    + Last Update: + {{pkg.metadata_modified}} + +
    +
    + {% if pkg.maintainer_email %} +
    + + + Contact dataset maintainer + +
    + {% endif %} + +
    +
    +
    + {% if pkg.featured_view %} +
    +

    Featured Visualization

    +
    + + + Fullscreen + +
    +
    + + {% endif %} +{% endblock %} + + +{% block secondary_content %} +
    + {% block secondary_help_content %} +
    +
    +

    {{ _('Activity') }}

    +
    +
    +
    + {#{ h.package_activity_html(pkg.id) | safe }#} + {% endblock %} + + {% block package_organization %} + {% if pkg.organization %} + {% set org = h.get_organization(pkg.organization.id) %} + {% snippet "snippets/organization.html", organization=org, has_context_title=true %} + {% endif %} + {% endblock %} + + + {% block package_info %} + + {% endblock %} + + {% block package_license %} + {% endblock %} +
    +{% endblock %} + {% block main_content %} {{ super() }}
    diff --git a/ckanext/iaea/templates/package/snippets/additional_info.html b/ckanext/iaea/templates/package/snippets/additional_info.html new file mode 100644 index 0000000..cf33483 --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/additional_info.html @@ -0,0 +1,90 @@ +
    + + + + + + + + + {% block package_additional_info %} + {% if pkg_dict.url %} + + + {% if h.is_url(pkg_dict.url) %} + + {% else %} + + {% endif %} + + {% endif %} + + {% if pkg_dict.author_email %} + + + + + {% elif pkg_dict.author %} + + + + + {% endif %} + + {% if pkg_dict.maintainer_email %} + + + + + {% elif pkg_dict.maintainer %} + + + + + {% endif %} + + {% if pkg_dict.version %} + + + + + {% endif %} + + {% if h.check_access('package_update',{'id':pkg_dict.id}) %} + + + + + {% endif %} + {% if pkg_dict.metadata_modified %} + + + + + {% endif %} + {% if pkg_dict.metadata_created %} + + + + + + {% endif %} + + {% block extras scoped %} + {% for extra in h.sorted_extras(pkg_dict.extras) %} + {% set key, value = extra %} + + + + + {% endfor %} + {% endblock %} + + {% endblock %} + +
    {{ _('Field') }}{{ _('Value') }}
    {{ _('Source') }}{{ h.link_to(pkg_dict.url, pkg_dict.url, rel='foaf:homepage', target='_blank') }}{{ pkg_dict.url }}
    {{ _("Author") }}{{ h.mail_to(email_address=pkg_dict.author_email, name=pkg_dict.author) }}
    {{ _("Author") }}{{ pkg_dict.author }}
    {{ _('Maintainer') }}{{ h.mail_to(email_address=pkg_dict.maintainer_email, name=pkg_dict.maintainer) }}
    {{ _('Maintainer') }}{{ pkg_dict.maintainer }}
    {{ _("Version") }}{{ pkg_dict.version }}
    {{ _("State") }}{{ _(pkg_dict.state) }}
    {{ _("Last Updated") }} + {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_modified %} +
    {{ _("Created") }} + {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_created %} +
    {{ _(key) }}{{ value }}
    +
    \ No newline at end of file diff --git a/ckanext/iaea/templates/package/snippets/view_form.html b/ckanext/iaea/templates/package/snippets/view_form.html new file mode 100644 index 0000000..6f6af3d --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/view_form.html @@ -0,0 +1,9 @@ +{% ckan_extends %} + +{% block view_form_filters %} + {% if h.resource_view_is_filterable(resource_view) %} + {% snippet 'package/snippets/view_form_filters.html', resource=resource, resource_view=resource_view %} + {% endif %} + {% snippet 'package/snippets/view_form_field_filter.html', resource=resource, resource_view=resource_view, data=data, errors=errors %} + {# resource 'iaea/view_js' #} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/snippets/view_form_field_filter.html b/ckanext/iaea/templates/package/snippets/view_form_field_filter.html new file mode 100644 index 0000000..4e815aa --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/view_form_field_filter.html @@ -0,0 +1,22 @@ +{% import 'macros/form.html' as form %} +{% macro suggestedFilter(name, fields, suggested_filter_fields, label = '', error = '', options = '', classes = [], is_required = false) %} + {% set classes = (classes | list) %} + {%- set extra_html = caller() if caller -%} + {% call form.input_block(id or name, label or name, error, classes, extra_html = extra_html, is_required = is_required) %} +
    + +
    + {% endcall %} +{% endmacro %} + +{% set fields = h.resource_view_get_fields(resource) %} +
    + {{ + suggestedFilter('Suggested filter fields', label=_('Suggested filter fields'), fields=fields, suggested_filter_fields=data.suggested_filter_fields, error=errors, classes=[""]) + }} +
    \ No newline at end of file diff --git a/ckanext/iaea/view.py b/ckanext/iaea/view.py new file mode 100644 index 0000000..cea03ca --- /dev/null +++ b/ckanext/iaea/view.py @@ -0,0 +1,197 @@ +import logging +import ckan.logic as logic +import ckan.lib.base as base +import ckan.model as model +from flask.views import MethodView +import ckan.lib.dictization.model_dictize as model_dictize +from itertools import groupby +import ckan.lib.helpers as h + +from ckan.common import _, c, request + +log = logging.getLogger(__name__) + +render = base.render +abort = base.abort + +NotFound = logic.NotFound +NotAuthorized = logic.NotAuthorized +ValidationError = logic.ValidationError +check_access = logic.check_access +get_action = logic.get_action + + +def metadata(id): # noqa + """Render this package's public activity stream page. + """ + context = { + u'model': model, + u'session': model.Session, + u'user': c.user, + u'for_view': True, + u'auth_user_obj': c.userobj + } + + data_dict = {u'id': id} + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + # TODO: remove + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/metadata.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + } + ) + + +class FeatureView(MethodView): + """ feature selected view on dataset page + """ + + def _prepare(self, id): + context = { + u'model': model, + u'session': model.Session, + u'user': c.user, + u'auth_user_obj': c.userobj, + u'save': u'save' in request.form + } + try: + check_access(u'package_update', context, {u'id': id}) + + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort( + 403, + _(u'Unauthorized to edit %s') % u'' + ) + + return context + + def post(self, id): + context = self._prepare(id) + + featured_view = request.form.get('featured_view', False) + data_dict = {u'id': id} + if request.form['submit'] == 'submit': + if not featured_view: + h.flash_error('Please select view from the list.') + data_dict['featured_view'] = '' + else: + data_dict['featured_view'] = featured_view + else: + data_dict['featured_view'] = '' + + # update package with selected featured view + try: + pkg_dict = get_action(u'package_patch')(context, data_dict) + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + package_views = model.Session.query( + model.ResourceView + ).join(model.Resource).filter(model.Resource.package_id == pkg_dict['id']).all() + + package_views_list = model_dictize.resource_view_list_dictize( + package_views, context) + + package_views_dict = [] + package_views_list = sorted(package_views_list, key=lambda k: k['resource_id']) + for k, v in groupby(package_views_list, key=lambda x: x['resource_id']): + [resource_name, state ] = model.Session.query(model.Resource.name.label('name'), model.Resource.state.label('state')).filter( + model.Resource.id == k).first() + + if state == 'deleted': + continue + else: + view_dict = { + 'resource_name': resource_name, + 'resource_id': k, + 'views': list(v) + } + + package_views_dict.append(view_dict) + + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/features_view.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + u'package_views': package_views_dict + }) + + def get(self, id): + context = self._prepare(id) + data_dict = {u'id': id} + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + package_views = model.Session.query( + model.ResourceView + ).join(model.Resource).filter(model.Resource.package_id == pkg_dict['id']).all() + + package_views_list = model_dictize.resource_view_list_dictize( + package_views, context) + + package_views_dict = [] + package_views_list = sorted(package_views_list, key=lambda k: k['resource_id']) + for k, v in groupby(package_views_list, key=lambda x: x['resource_id']): + [resource_name, state ] = model.Session.query(model.Resource.name.label('name'), model.Resource.state.label('state')).filter( + model.Resource.id == k).first() + + if state == 'deleted': + continue + else: + view_dict = { + 'resource_name': resource_name, + 'resource_id': k, + 'views': list(v) + } + + package_views_dict.append(view_dict) + + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/features_view.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + u'package_views': package_views_dict + }) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..90a0123 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,108 @@ +version: "3" + +services: + ckan: + profiles: ["full"] + build: + context: docker/ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + env_file: + - .env + links: + - db + - solr + - redis + ports: + - "${CKAN_PORT}:5000" + volumes: + #- .:/srv/app/src_extensions/ckanext-iaea + - ckan_storage:/var/lib/ckan + + # ckan-worker-default: + # build: + # context: docker/ckan/ + # dockerfile: Dockerfile + # args: + # - TZ=${TZ} + # env_file: + # - .env + # links: + # - db + # - solr + # - redis + # volumes: + # - .:/srv/app/src_extensions/ckanext-iaea + # - ckan_storage:/var/lib/ckan + # command: ["/srv/app/start_ckan_worker.sh", "default"] + + # ckan-worker-bulk: + # build: + # context: docker/ckan/ + # dockerfile: Dockerfile + # args: + # - TZ=${TZ} + # env_file: + # - .env + # links: + # - db + # - solr + # - redis + # volumes: + # - .:/srv/app/src_extensions/ckanext-iaea + # - ckan_storage:/var/lib/ckan + # command: ["/srv/app/start_ckan_worker.sh", "bulk"] + + # ckan-worker-priority: + # build: + # context: docker/ckan/ + # dockerfile: Dockerfile + # args: + # - TZ=${TZ} + # env_file: + # - .env + # links: + # - db + # - solr + # - redis + # volumes: + # - .:/srv/app/src_extensions/ckanext-iaea + # - ckan_storage:/var/lib/ckan + # command: ["/srv/app/start_ckan_worker.sh", "priority"] + + db: + container_name: iaea_db + env_file: + - .env + environment: + - "PGDATA=/var/lib/postgresql/data/dbdata" + ports: + - "5432:5432" + build: + context: docker/postgresql/ + volumes: + - pg_data:/var/lib/postgresql/data/dbdata + + solr: + image: ckan/ckan-solr:master + container_name: iaea_solr + ulimits: + nofile: + soft: 65536 + hard: 65536 + ports: + - "8983:8983" + volumes: + - solr_data:/opt/solr/server/solr/ckan/data/index + + redis: + container_name: iaea_redis + image: redis:alpine + ports: + - "6379:6379" + +volumes: + ckan_storage: + pg_data: + solr_data: \ No newline at end of file diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile new file mode 100644 index 0000000..e1dc293 --- /dev/null +++ b/docker/ckan/Dockerfile @@ -0,0 +1,75 @@ +FROM ckan/ckan-base:2.9 + +MAINTAINER Open Knowledge International + +ENV APP_DIR=/srv/app +ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions +ENV IAEA_EXTENSION_DIR=/srv/app/src_extensions/ckanext-iaea + +# Set timezone +ARG TZ +RUN cp /usr/share/zoneinfo/$TZ /etc/localtime +RUN echo $TZ > /etc/timezone + +# Install packages needed by the dev requirements +RUN apk add --no-cache libffi-dev + +# # Install CKAN dev requirements +# #RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt +RUN pip install flask-debugtoolbar + + +# Install CKAN extensions + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-xloader@rewrite-site-url#egg=ckanext-xloader" && \ + pip install -r "./src/ckanext-xloader/requirements.txt" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-validation@iaea-2.9#egg=ckanext-validation" && \ + pip install -r "./src/ckanext-validation/requirements.txt" + +RUN pip install -e "git+https://github.com/ckan/ckanext-report#egg=ckanext-report" && \ + pip install -r "./src/ckanext-report/requirements.txt" + +RUN pip install -e "git+https://github.com/ckan/ckanext-archiver#egg=ckanext-archiver" && \ + sed -i '/ckanext-report/d' "./src/ckanext-archiver/requirements.txt" && \ + pip install -r "./src/ckanext-archiver/requirements.txt" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-qa@qa-iaea#egg=ckanext-qa" && \ + pip install -r "./src/ckanext-qa/requirements.txt" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-harvest@montreal-fix#egg=ckanext-harvest" && \ + pip install -r "./src/ckanext-harvest/pip-requirements.txt" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-basiccharts@iaea#egg=ckanext-basiccharts" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-visualize#egg=ckanext-visualize" && \ + pip install -r "./src/ckanext-visualize/requirements.txt" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-pdfview#egg=ckanext-pdfview" + +RUN pip install -e "git+https://github.com/ckan/ckanext-geoview@v0.1.0#egg=ckanext-geoview" + +RUN pip install -e "git+https://github.com/ckan/ckanext-pages@v0.5.2#egg=ckanext-pages" && \ + pip install -r "./src/ckanext-pages/requirements.txt" + +RUN pip install -e "git+https://github.com/datopian/ckanext-dataexplorer-react#egg=ckanext-dataexplorer-react" && \ + pip install -r "./src/ckanext-dataexplorer-react/requirements.txt" + +RUN pip install -e "git+https://github.com/ckan/ckanext-dcat@v1.5.1#egg=ckanext-dcat" && \ + pip install -r "./src/ckanext-dcat/requirements.txt" + +RUN pip install -e "git+https://github.com/ckan/ckanext-scheming@release-3.0.0#egg=ckanext-scheming" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-saml2auth@ckan-2.9-iaea#egg=ckanext-saml2auth" + + +# Create folder for local extensions sources +RUN mkdir $SRC_EXTENSIONS_DIR +RUN mkdir $IAEA_EXTENSION_DIR + +COPY setup/start_ckan_development.sh ${APP_DIR} +COPY setup/start_ckan_worker.sh ${APP_DIR} +COPY ckan.ini ${APP_DIR} + + +CMD ["/srv/app/start_ckan_development.sh"] diff --git a/docker/ckan/ckan.ini b/docker/ckan/ckan.ini new file mode 100644 index 0000000..546fe88 --- /dev/null +++ b/docker/ckan/ckan.ini @@ -0,0 +1,258 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 5000 + + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = LpgiJSdPY7uSMEiPHvX8jJzYA + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = f296bd5f-cff8-4180-b6a9-3f87bbfc4fbb + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +# who.timeout = 86400 + +## Database Settings +sqlalchemy.url = postgresql://ckan:ckan@iaea_db/ckan + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = http://ckan.iaea.local:5000 +#ckan.use_pylons_response_cleanup_middleware = true + +# Default timeout for Requests +#ckan.requests.timeout = 10 + + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = false +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = true +ckan.auth.public_activity_stream_detail = true +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:LpgiJSdPY7uSMEiPHvX8jJzYA +api_token.jwt.decode.secret = string:LpgiJSdPY7uSMEiPHvX8jJzYA +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default +#solr_url = http://127.0.0.1:8983/solr + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +#ckan.redis.url = redis://localhost:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +ckan.plugins = envvars stats text_view image_view recline_view + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +#ckan.storage_path = /var/lib/ckan +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 +# Default timeout for fetching proxied items +#ckan.resource_proxy.timeout = 10 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/docker/ckan/setup/start_ckan_development.sh b/docker/ckan/setup/start_ckan_development.sh new file mode 100755 index 0000000..0f31a9c --- /dev/null +++ b/docker/ckan/setup/start_ckan_development.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Install any local extensions in the src_extensions volume +echo "Looking for local extensions to install..." +echo "Extension dir contents:" +ls -la $SRC_EXTENSIONS_DIR +for i in $SRC_EXTENSIONS_DIR/* +do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python3 $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi +done + +# Set debug to true +echo "Enabling debug mode" +ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + +# Update the plugins setting in the ini file with the values defined in the env var +echo "Loading the following plugins: $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" + +ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" + +# Update test-core.ini DB, SOLR & Redis settings +echo "Loading test settings into test-core.ini" +ckan config-tool $SRC_DIR/ckan/test-core.ini \ + "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ + "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ + "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ + "solr_url = $TEST_CKAN_SOLR_URL" \ + "ckan.redis.url = $TEST_CKAN_REDIS_URL" + +# Run the prerun script to init CKAN and create the default admin user +python3 prerun.py + +echo "Running DB init scripts" +# paster --plugin=ckanext-archiver archiver init --config="$CKAN_INI" +# paster --plugin=ckanext-report report initdb --config="$CKAN_INI" +# paster --plugin=ckanext-qa qa init --config="$CKAN_INI" +# paster --plugin=ckanext-validation validation init-db --config="$CKAN_INI" +echo "Init DB scripts completed." + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Start supervisord +# supervisord --configuration /etc/supervisord.conf & + +# Start the development server with automatic reload +ckan -c "${CKAN_INI}" run diff --git a/docker/ckan/setup/start_ckan_worker.sh b/docker/ckan/setup/start_ckan_worker.sh new file mode 100755 index 0000000..53cb54e --- /dev/null +++ b/docker/ckan/setup/start_ckan_worker.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Install any local extensions in the src_extensions volume +echo "Looking for local extensions to install..." +echo "Extension dir contents:" +ls -la $SRC_EXTENSIONS_DIR +for i in $SRC_EXTENSIONS_DIR/* +do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python3 $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi +done + + +# Set debug to true +echo "Enabling debug mode" +ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + +# Update the plugins setting in the ini file with the values defined in the env var +echo "Loading the following plugins: $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" + +ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" + +echo "Running worker: $1" +ckan jobs worker "$1" -c "$CKAN_INI" \ No newline at end of file diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile new file mode 100644 index 0000000..ebb6341 --- /dev/null +++ b/docker/postgresql/Dockerfile @@ -0,0 +1,14 @@ +FROM postgres:14.11-alpine +MAINTAINER Open Knowledge International + +# Allow connections; we don't map out any ports so only linked docker containers can connect +RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf + +# Customize default user/pass/db +ENV POSTGRES_DB ckan +ENV POSTGRES_USER ckan +ARG POSTGRES_PASSWORD +ARG DATASTORE_READONLY_PASSWORD + +# Include extra setup scripts (eg datastore) +ADD docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql b/docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql new file mode 100644 index 0000000..a811507 --- /dev/null +++ b/docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql @@ -0,0 +1,4 @@ +\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\'' + +CREATE ROLE datastore_ro NOSUPERUSER LOGIN PASSWORD :datastore_ro_password; +CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; diff --git a/docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql b/docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql new file mode 100644 index 0000000..140f2e5 --- /dev/null +++ b/docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql @@ -0,0 +1,2 @@ +CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8'; +CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8';