It will not be obtainable later,'
+ + ' so please copy it now before clicking away.'
+ + ' (You can generate a new one if you lose this'
+ + ' one.). Your API token will expire on'
+ + ' ' + this.expirationDate + '
'
+ );
+ this.btnGenerate.hide();
+ this.panelCopyDelete.show();
+ this.prepareToSyncWindowShadow();
+ this.parentWindow.setWidth(550);
+ },
+
+ processSuccessfulResponse: function (params, response) {
+ var data = CCR.safelyDecodeJSONResponse(response);
+ if (CCR.checkDecodedJSONResponseSuccess(data)) {
+ params.onSuccess(this, data);
+ } else if (params.onFailure) {
+ params.onFailure(this, response.status);
+ } else {
+ this.showServerErrorMsg(params.actionText);
+ }
+ },
+
+ showServerErrorMsg: function (actionText) {
+ Ext.Msg.show({
+ title: 'Server Error',
+ msg: this.getServerErrorMsg(actionText),
+ icon: Ext.MessageBox.ERROR,
+ buttons: Ext.Msg.OK,
+ minWidth: this.EXT_MSG_MIN_WIDTH,
+ maxWidth: this.EXT_MSG_MAX_WIDTH
+ });
+ },
+
+ showReceivedToken: function () {
+ var isTokenExpired = (
+ new Date(this.creationDate)
+ >= new Date(this.expirationDate)
+ );
+ if (isTokenExpired) {
+ this.showMsg(this.getExpiredTokenMsg());
+ } else {
+ this.showMsg(this.getUnexpiredTokenMsg());
+ }
+ this.btnRetry.hide();
+ this.btnDeleteOld.show();
+ XDMoD.utils.syncWindowShadow(this);
+ },
+
+ showMsg: function (msg) {
+ this.updateHtml(this.panelMsg, msg);
+ },
+
+ processGetRequestServerError: function (self) {
+ self.showMsg(self.getServerErrorMsg(self.ACTION_TEXT_FOR_GET));
+ self.btnRetry.show();
+ XDMoD.utils.syncWindowShadow(self);
+ },
+
+ confirmRequest: function (params) {
+ var self = this;
+ Ext.Msg.show({
+ title: params.text,
+ msg: params.confirmMsg,
+ icon: Ext.MessageBox.QUESTION,
+ buttons: Ext.Msg.YESNO,
+ minWidth: this.EXT_MSG_MIN_WIDTH,
+ maxWidth: this.EXT_MSG_MAX_WIDTH,
+ fn: function (resp) {
+ if (resp === 'yes') {
+ self.request(params);
+ }
+ }
+ });
+ },
+
+ updateHtml: function (obj, html) {
+ if (obj.rendered) {
+ obj.update(html);
+ } else {
+ obj.html = html;
+ }
+ },
+
+ prepareToSyncWindowShadow: function () {
+ this.parentWindow.addListener(
+ 'resize',
+ function () {
+ XDMoD.utils.syncWindowShadow(this);
+ },
+ this, {
+ delay: 50,
+ single: true
+ }
+ );
+ },
+
+ getServerErrorMsg: function (actionText) {
+ return 'Server error ' + actionText + '. Please try again.';
+ },
+
+ getExpiredTokenMsg: function () {
+ return (
+ 'Your API token expired on ' + this.expirationDate + ''
+ + ' Please delete it and generate a new one.'
+ );
+ },
+
+ getUnexpiredTokenMsg: function () {
+ return (
+ 'Your API token was generated on'
+ + ' ' + this.creationDate + ''
+ + ' and expires on'
+ + ' ' + this.expirationDate + ''
+ + ' If you have lost your API token,'
+ + ' please delete it and generate a new one.'
+ );
+ }
+});
diff --git a/html/gui/js/profile_editor/ProfileEditor.js b/html/gui/js/profile_editor/ProfileEditor.js
index 9e84f4e923..962718b947 100644
--- a/html/gui/js/profile_editor/ProfileEditor.js
+++ b/html/gui/js/profile_editor/ProfileEditor.js
@@ -1,94 +1,93 @@
XDMoD.Profile = {
- logoutOnClose: false
+ logoutOnClose: false
};
XDMoD.ProfileEditorConstants = {
- PASSWORD: 0,
+ PASSWORD: 0,
WELCOME_EMAIL_CHANGE: 1, // designates if we're displaying first time login prompt to validate email
SSO_USER: 5 // designates whether or not this is a Single Sign On user
-};//XDMoD.ProfileEditorConstants
+}; // XDMoD.ProfileEditorConstants
// --------------------------------------------
-XDMoD.ProfileEditor = Ext.extend(Ext.Window, {
+XDMoD.ProfileEditor = Ext.extend(Ext.Window, {
id: 'xdmod-profile-editor',
- width:375,
+ width: 375,
- border:false,
- frame: true,
+ border: false,
+ frame: true,
- iconCls: 'user_profile_16',
+ iconCls: 'user_profile_16',
- modal:true,
- closable:true,
+ modal: true,
+ closable: true,
- closeAction:'close',
- resizable:false,
+ closeAction: 'close',
+ resizable: false,
- title:'My Profile',
+ title: 'My Profile',
- tooltip: 'Profile Editor',
+ tooltip: 'Profile Editor',
- init: function() {
- this.general_settings.init();
- },
+ init: function () {
+ this.general_settings.init();
+ this.api_token.init();
+ },
- handleProfileClose: function() {
+ handleProfileClose: function () {
+ if (XDMoD.Profile.logoutOnClose) {
+ Ext.Msg.show({
+ maxWidth: 800,
+ minWidth: 400,
+ title: 'Close profile and logout?',
+ msg: 'If you do not supply an e-mail address, you will be logged out of XDMoD.
Are you sure you want to do this?',
+ buttons: Ext.Msg.YESNO,
- if (XDMoD.Profile.logoutOnClose == true) {
+ fn: function (resp) {
+ if (resp === 'yes') {
+ CCR.xdmod.ui.actionLogout();
+ }
+ }, // fn
- Ext.Msg.show({
+ icon: Ext.MessageBox.QUESTION
+ }); // Ext.Msg.show
- maxWidth: 800,
- minWidth: 400,
- title: 'Close profile and logout?',
- msg: 'If you do not supply an e-mail address, you will be logged out of XDMoD.
Are you sure you want to do this?',
- buttons: Ext.Msg.YESNO,
+ return false;
+ } // if (XDMoD.Profile.logoutOnClose)
- fn: function(resp) {
+ return true;
+ },
- if (resp == 'yes')
- CCR.xdmod.ui.actionLogout();
+ getCloseButton: function () {
+ var self = this;
- },//fn
-
- icon: Ext.MessageBox.QUESTION
-
- });//Ext.Msg.show
-
- return false;
-
- }//if (XDMoD.Profile.logoutOnClose == true)
-
- return true;
-
- },
-
- getCloseButton: function() {
-
- var self = this;
-
- return new Ext.Button({
- text: 'Close',
- iconCls: 'general_btn_close',
- handler: function(){ self.close(); }
- });
-
- },
+ return new Ext.Button({
+ text: 'Close',
+ iconCls: 'general_btn_close',
+ handler: function () {
+ self.close();
+ }
+ });
+ },
- initComponent: function(){
+ initComponent: function () {
+ var self = this;
- var self = this;
+ this.general_settings = new XDMoD.ProfileGeneralSettings({
+ parentWindow: self
+ });
- this.general_settings = new XDMoD.ProfileGeneralSettings({parentWindow: self});
+ this.api_token = new XDMoD.ProfileApiToken({
+ parentWindow: self
+ });
- // ------------------------------------------------
+ // ------------------------------------------------
- this.on('beforeclose', self.handleProfileClose);
+ this.on('beforeclose', self.handleProfileClose);
- // ------------------------------------------------
+ // ------------------------------------------------
var tabItems = [
this.general_settings
@@ -111,7 +110,7 @@ XDMoD.ProfileEditor = Ext.extend(Ext.Window, {
items: [{
title: 'User Interface',
xtype: 'form',
- bodyStyle: 'padding:5px',
+ bodyStyle: 'padding: 5px',
labelWidth: 230,
frame: true,
items: [{
@@ -143,53 +142,49 @@ XDMoD.ProfileEditor = Ext.extend(Ext.Window, {
});
}
- var tabPanel = new Ext.TabPanel({
+ tabItems.push(this.api_token);
- frame: false,
- border: false,
- activeTab: 0,
+ var tabPanel = new Ext.TabPanel({
+ frame: false,
+ border: false,
+ activeTab: 0,
- defaults: {
- tabCls: 'tab-strip'
- },
+ defaults: {
+ tabCls: 'tab-strip'
+ },
- items: tabItems,
+ items: tabItems,
- listeners: {
- tabchange: function (thisTabPanel, tab) {
- XDMoD.utils.syncWindowShadow(thisTabPanel);
+ listeners: {
+ tabchange: function (thisTabPanel, tab) {
+ XDMoD.utils.syncWindowShadow(thisTabPanel);
- // Fix a bug where invalid field errors are displayed poorly
- // after a tab switch by forcing form validation.
- if (tab !== self.general_settings) {
- return;
- }
+ // Fix a bug where invalid field errors are displayed poorly
+ // after a tab switch by forcing form validation.
+ if (tab !== self.general_settings) {
+ return;
+ }
- tab.cascade(function (currentComponent) {
- if (currentComponent instanceof Ext.form.FormPanel) {
- currentComponent.getForm().isValid();
- return false;
- }
+ tab.cascade(function (currentComponent) {
+ if (currentComponent instanceof Ext.form.FormPanel) {
+ currentComponent.getForm().isValid();
+ return false;
+ }
- return true;
- });
+ return true;
+ });
+ }
}
- }
-
- });//tabPanel
-
- // ------------------------------------------------
-
- Ext.apply(this, {
-
- items:[
- tabPanel
- ]
-
- });
+ }); // tabPanel
- XDMoD.ProfileEditor.superclass.initComponent.call(this);
+ // ------------------------------------------------
- }//initComponent
+ Ext.apply(this, {
+ items: [
+ tabPanel
+ ]
+ });
-});//XDMoD.ProfileEditor
+ XDMoD.ProfileEditor.superclass.initComponent.call(this);
+ } // initComponent
+}); // XDMoD.ProfileEditor
diff --git a/html/gui/js/profile_editor/ProfileGeneralSettings.js b/html/gui/js/profile_editor/ProfileGeneralSettings.js
index aeb4f14ad1..458de49f04 100644
--- a/html/gui/js/profile_editor/ProfileGeneralSettings.js
+++ b/html/gui/js/profile_editor/ProfileGeneralSettings.js
@@ -302,7 +302,6 @@ XDMoD.ProfileGeneralSettings = Ext.extend(Ext.form.FormPanel, {
frame: true,
title: 'User Information',
bodyStyle: 'padding:5px 5px 0',
- width: 350,
defaults: {
width: 200
},
@@ -330,7 +329,6 @@ XDMoD.ProfileGeneralSettings = Ext.extend(Ext.form.FormPanel, {
frame: true,
title: 'Update Password',
bodyStyle: 'padding:5px 5px 0',
- width: 350,
defaults: {
width: 200
},
@@ -353,7 +351,6 @@ XDMoD.ProfileGeneralSettings = Ext.extend(Ext.form.FormPanel, {
labelWidth: 95,
frame: false,
bodyStyle: 'padding:0px 5px',
- width: 350,
layout: 'form',
items: [{
xtype: 'tbtext',
@@ -372,7 +369,6 @@ XDMoD.ProfileGeneralSettings = Ext.extend(Ext.form.FormPanel, {
labelWidth: 95,
frame: false,
bodyStyle: 'padding:0px 5px',
- width: 350,
layout: 'form',
items: [{
xtype: 'tbtext',
diff --git a/html/index.php b/html/index.php
index 1dfa746786..689536eeac 100644
--- a/html/index.php
+++ b/html/index.php
@@ -384,6 +384,7 @@ function ($item) {
+
diff --git a/tests/artifacts/xdmod/integration/base/date.spec.json b/tests/artifacts/xdmod/integration/base/date.spec.json
new file mode 100644
index 0000000000..5c828eab26
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/base/date.spec.json
@@ -0,0 +1,5 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "string",
+ "pattern": "[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}"
+}
diff --git a/tests/artifacts/xdmod/integration/base/error.json b/tests/artifacts/xdmod/integration/base/error.json
new file mode 100644
index 0000000000..4e600b5109
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/base/error.json
@@ -0,0 +1,10 @@
+{
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "",
+ "code": 0
+}
diff --git a/tests/artifacts/xdmod/integration/base/success.spec.json b/tests/artifacts/xdmod/integration/base/success.spec.json
new file mode 100644
index 0000000000..f89fb09e2f
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/base/success.spec.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean",
+ "enum": [ true ]
+ }
+ },
+ "required": [
+ "success"
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json b/tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json
new file mode 100644
index 0000000000..998af806bb
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json
@@ -0,0 +1,152 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "totalCount": {
+ "type": "number",
+ "enum": [ 1 ]
+ },
+ "data": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/data_item" },
+ "minItems": 1,
+ "maxItems": 1,
+ "uniqueItems": true
+ }
+ },
+ "required": [ "totalCount", "data" ],
+ "additionalProperties": false,
+ "$defs": {
+ "data_item": {
+ "type": "object",
+ "properties": {
+ "realms": { "$ref": "#/$defs/realms" }
+ },
+ "required": [ "realms" ],
+ "additionalProperties": false
+ },
+ "realms": {
+ "type": "object",
+ "properties": {
+ "Cloud": {
+ "allOf": [
+ { "$ref": "#/$defs/realm" },
+ { "$ref": "#/$defs/cloud_realm" }
+ ]
+ },
+ "Jobs": {
+ "allOf" : [
+ { "$ref": "#/$defs/realm" },
+ { "$ref": "#/$defs/jobs_realm" }
+ ]
+ },
+ "Storage": {
+ "allOf" : [
+ { "$ref": "#/$defs/realm" },
+ { "$ref": "#/$defs/storage_realm" }
+ ]
+ }
+ },
+ "required": [ "Cloud", "Jobs", "Storage" ],
+ "additionalProperties": false
+ },
+ "realm": {
+ "type": "object",
+ "properties": {
+ "metrics": { "$ref": "#/$defs/metrics" },
+ "dimensions": { "$ref": "#/$defs/dimensions" },
+ "text": { "type": "string" },
+ "category": { "type": "string" }
+ },
+ "required": [ "metrics", "dimensions", "text", "category" ],
+ "additionalProperties": false
+ },
+ "cloud_realm": {
+ "properties": {
+ "metrics": {
+ "minProperties": 10,
+ "maxProperties": 10
+ },
+ "dimensions": {
+ "minProperties": 14,
+ "maxProperties": 14
+ },
+ "text": {
+ "enum": [ "Cloud" ]
+ },
+ "category": {
+ "enum": [ "Cloud" ]
+ }
+ }
+ },
+ "jobs_realm": {
+ "properties": {
+ "metrics": {
+ "minProperties": 26,
+ "maxProperties": 26
+ },
+ "dimensions": {
+ "minProperties": 16,
+ "maxProperties": 16
+ },
+ "text": {
+ "enum": [ "Jobs" ]
+ },
+ "category": {
+ "enum": [ "Jobs" ]
+ }
+ }
+ },
+ "storage_realm": {
+ "properties": {
+ "metrics": {
+ "minProperties": 7,
+ "maxProperties": 7
+ },
+ "dimensions": {
+ "minProperties": 10,
+ "maxProperties": 10
+ },
+ "text": {
+ "enum": [ "Storage" ]
+ },
+ "category": {
+ "enum": [ "Storage" ]
+ }
+ }
+ },
+ "metrics": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ { "$ref": "#/$defs/metric_or_dimension" },
+ {
+ "properties": {
+ "std_err": { "type": "boolean" }
+ }
+ }
+ ]
+ }
+ }
+ },
+ "dimensions": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": { "$ref": "#/$defs/metric_or_dimension" }
+ }
+ },
+ "metric_or_dimension": {
+ "type": "object",
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "info": {
+ "type": "string"
+ }
+ },
+ "required": [ "text", "info" ]
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json
new file mode 100644
index 0000000000..f778eee9dc
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "An error was encountered while attempting to process the requested authorization procedure."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json
new file mode 100644
index 0000000000..77705326ed
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "Token already exists."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json
new file mode 100644
index 0000000000..a5e81d10e7
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json
@@ -0,0 +1,24 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "${INTEGRATION_ROOT}/base/success.spec.json" },
+ {
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "token": {
+ "type": "string",
+ "pattern": "^[0-9]+\\.[0-9a-f]{64}$"
+ },
+ "expiration_date": {
+ "$ref": "${INTEGRATION_ROOT}/base/date.spec.json"
+ }
+ },
+ "required": [ "token", "expiration_date" ]
+ }
+ },
+ "required": [ "data" ]
+ }
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json
new file mode 100644
index 0000000000..f4be0727bb
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "No Token Provided."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json
new file mode 100644
index 0000000000..8c028c39d8
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "The API Token has expired."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json
new file mode 100644
index 0000000000..34cd7b0c39
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json
@@ -0,0 +1,23 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "${INTEGRATION_ROOT}/base/success.spec.json" },
+ {
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "created_on": {
+ "$ref": "${INTEGRATION_ROOT}/base/date.spec.json"
+ },
+ "expiration_date": {
+ "$ref": "${INTEGRATION_ROOT}/base/date.spec.json"
+ }
+ },
+ "required": [ "created_on", "expiration_date" ]
+ }
+ },
+ "required": [ "data" ]
+ }
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json
new file mode 100644
index 0000000000..6ad39a9032
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "Invalid API token."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json
new file mode 100644
index 0000000000..a93df9acc9
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "Invalid token format."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json
new file mode 100644
index 0000000000..133ce7170e
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "API token not found."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json
new file mode 100644
index 0000000000..f918ae563c
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json
@@ -0,0 +1,4 @@
+{
+ "success": true,
+ "message": "Token successfully revoked."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json
new file mode 100644
index 0000000000..a95387487b
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json
@@ -0,0 +1,5 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "Session Expired",
+ "code": 2
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json
deleted file mode 100644
index ddf9e65474..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json
+++ /dev/null
@@ -1,83 +0,0 @@
-{
- "Normal user": [
- "usr",
- 200,
- "GET-realms",
- [
- {
- "id": "Jobs",
- "name": "Jobs"
- },
- {
- "id": "Cloud",
- "name": "Cloud"
- }
- ]
- ],
- "PI": [
- "pi",
- 200,
- "GET-realms",
- [
- {
- "id": "Jobs",
- "name": "Jobs"
- },
- {
- "id": "Cloud",
- "name": "Cloud"
- }
- ]
- ],
- "Center staff": [
- "cs",
- 200,
- "GET-realms",
- [
- {
- "id": "Jobs",
- "name": "Jobs"
- },
- {
- "id": "Cloud",
- "name": "Cloud"
- }
- ]
- ],
- "Center director": [
- "cd",
- 200,
- "GET-realms",
- [
- {
- "id": "Jobs",
- "name": "Jobs"
- },
- {
- "id": "Cloud",
- "name": "Cloud"
- }
- ]
- ],
- "Administrative user": [
- "mgr",
- 200,
- "GET-realms",
- [
- {
- "id": "Jobs",
- "name": "Jobs"
- },
- {
- "id": "Cloud",
- "name": "Cloud"
- }
- ]
- ],
- "Public user": [
- "pub",
- 401,
- "error",
- []
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/input/create-request.json
similarity index 100%
rename from tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json
rename to tests/artifacts/xdmod/integration/rest/warehouse/export/input/create-request.json
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/input/delete-request.json
similarity index 100%
rename from tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json
rename to tests/artifacts/xdmod/integration/rest/warehouse/export/input/delete-request.json
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/input/delete-requests.json
similarity index 100%
rename from tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json
rename to tests/artifacts/xdmod/integration/rest/warehouse/export/input/delete-requests.json
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/input/get-request.json
similarity index 100%
rename from tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json
rename to tests/artifacts/xdmod/integration/rest/warehouse/export/input/get-request.json
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/input/get-requests.json
similarity index 100%
rename from tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json
rename to tests/artifacts/xdmod/integration/rest/warehouse/export/input/get-requests.json
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json
new file mode 100644
index 0000000000..3ad5f561be
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json
@@ -0,0 +1,100 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean",
+ "enum": [ true ]
+ },
+ "data": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/realm" },
+ "minItems": 2,
+ "maxItems": 2,
+ "uniqueItems": true
+ },
+ "total": {
+ "type": "number",
+ "enum": [ 2 ]
+ }
+ },
+ "required": [ "success", "data", "total" ],
+ "additionalProperties": false
+ },
+ { "$ref": "#/$defs/jobs_realm" },
+ { "$ref": "#/$defs/cloud_realm" }
+ ],
+ "$defs": {
+ "realm": {
+ "type": "object",
+ "properties": {
+ "id": { "type": "string" },
+ "name": { "type": "string" },
+ "fields": { "$ref": "#/$defs/fields" }
+ },
+ "required": [ "id", "name", "fields" ],
+ "additionalProperties": false
+ },
+ "jobs_realm": {
+ "properties": {
+ "data": {
+ "contains": {
+ "properties": {
+ "id": { "enum": [ "Jobs" ] },
+ "name": { "enum": [ "Jobs" ] },
+ "fields": {
+ "minItems": 28,
+ "maxItems": 28
+ }
+ }
+ },
+ "minContains": 1,
+ "maxContains": 1
+ }
+ }
+ },
+ "cloud_realm": {
+ "properties": {
+ "data": {
+ "contains": {
+ "properties": {
+ "id": { "enum": [ "Cloud" ] },
+ "name": { "enum": [ "Cloud" ] },
+ "fields": {
+ "minItems": 16,
+ "maxItems": 16
+ }
+ }
+ },
+ "minContains": 1,
+ "maxContains": 1
+ }
+ }
+ },
+ "fields": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/field" },
+ "uniqueItems": true
+ },
+ "field": {
+ "type": "object",
+ "properties": {
+ "name": { "type": "string" },
+ "alias": { "type": "string" },
+ "display": { "type": "string" },
+ "anonymize": { "type": "boolean" },
+ "documentation": { "type": "string" }
+ },
+ "required": [
+ "name",
+ "alias",
+ "display",
+ "anonymize",
+ "documentation"
+ ],
+ "additionalProperties": false
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/schema/integration/authentication_error.spec.json b/tests/artifacts/xdmod/schema/integration/authentication_error.spec.json
deleted file mode 100644
index 13017e08fa..0000000000
--- a/tests/artifacts/xdmod/schema/integration/authentication_error.spec.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "authentication_error",
- "type": "object",
- "properties": {
- "success": {
- "enum": [
- false
- ]
- },
- "count": {
- "type": "integer"
- },
- "total": {
- "type": "integer"
- },
- "totalCount": {
- "type": "integer"
- },
- "results": {
- "type": "array"
- },
- "data": {
- "type": "array"
- },
- "message": {
- "const": "An error was encountered while attempting to process the requested authorization procedure."
- },
- "code": {
- "type": "integer"
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/create_user_token_failure.spec.json b/tests/artifacts/xdmod/schema/integration/create_user_token_failure.spec.json
deleted file mode 100644
index 5f36bbd419..0000000000
--- a/tests/artifacts/xdmod/schema/integration/create_user_token_failure.spec.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "create_user_token_failure",
- "type": "object",
- "properties": {
- "success": {
- "enum": [false]
- },
- "message": {
- "enum": [
- "Expiration Interval not provided.",
- "Unable to create a new API token.",
- "Unable to create a new token at this time."
- ]
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/create_user_token_success.spec.json b/tests/artifacts/xdmod/schema/integration/create_user_token_success.spec.json
deleted file mode 100644
index c532a2ecb2..0000000000
--- a/tests/artifacts/xdmod/schema/integration/create_user_token_success.spec.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "create_user_token_success",
- "type": "object",
- "properties": {
- "success": {
- "type": "boolean"
- },
- "data": {
- "type": "object",
- "properties": {
- "token": {
- "type": "string"
- },
- "expiration_date": {
- "type": "string"
- }
- }
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/get_user_token_failure.spec.json b/tests/artifacts/xdmod/schema/integration/get_user_token_failure.spec.json
deleted file mode 100644
index 4f028e0728..0000000000
--- a/tests/artifacts/xdmod/schema/integration/get_user_token_failure.spec.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "get_user_token_failure",
- "type": "object",
- "properties": {
- "success": {
- "enum": [false]
- },
- "message": {
- "const": "Unable to retrieve current API token."
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/get_user_token_success.spec.json b/tests/artifacts/xdmod/schema/integration/get_user_token_success.spec.json
deleted file mode 100644
index 52e7886e55..0000000000
--- a/tests/artifacts/xdmod/schema/integration/get_user_token_success.spec.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "get_user_token_success",
- "type": "object",
- "properties": {
- "success": {
- "enum": [true]
- },
- "data": {
- "type": "object",
- "properties": {
- "created_on": {
- "type": "string"
- },
- "expires_on": {
- "type": "string"
- }
- }
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/revoke_user_token_failure.spec.json b/tests/artifacts/xdmod/schema/integration/revoke_user_token_failure.spec.json
deleted file mode 100644
index 26a0566de5..0000000000
--- a/tests/artifacts/xdmod/schema/integration/revoke_user_token_failure.spec.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "revoke_user_token_failure",
- "type": "object",
- "properties": {
- "success": {
- "enum": [false]
- },
- "message": {
- "enum": [
- "Unable to revoke API Token.",
- "Unable to successfully revoke token.",
- "No token to revoke."
-
- ]
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/revoke_user_token_success.spec.json b/tests/artifacts/xdmod/schema/integration/revoke_user_token_success.spec.json
deleted file mode 100644
index 67eeccfbcb..0000000000
--- a/tests/artifacts/xdmod/schema/integration/revoke_user_token_success.spec.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "revoke_user_token_success",
- "type": "object",
- "properties": {
- "success": {
- "enum": [true]
- },
- "message": {
- "const": "Token successfully revoked."
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/integration/session_expired.spec.json b/tests/artifacts/xdmod/schema/integration/session_expired.spec.json
deleted file mode 100644
index ff800aa6ff..0000000000
--- a/tests/artifacts/xdmod/schema/integration/session_expired.spec.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "title": "authentication_error",
- "type": "object",
- "properties": {
- "success": {
- "type": "boolean",
- "enum": [
- false
- ]
- },
- "count": {
- "type": "integer",
- "const": 0
- },
- "total": {
- "type": "integer",
- "const": 0
- },
- "totalCount": {
- "type": "integer",
- "const": 0
- },
- "results": {
- "type": "array"
- },
- "data": {
- "type": "array"
- },
- "message": {
- "const": "Session Expired"
- },
- "code": {
- "type": "integer",
- "enum": [2]
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json
deleted file mode 100644
index 7ed2ce8abf..0000000000
--- a/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json
+++ /dev/null
@@ -1,75 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Response from GET rest/warehouse/export/realms",
- "type": "object",
- "properties": {
- "success": {
- "type": "boolean"
- },
- "data": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/realm"
- }
- },
- "total": {
- "type": "integer"
- }
- },
- "required": [
- "success",
- "data"
- ],
- "additionalProperties": false,
- "definitions": {
- "realm": {
- "type": "object",
- "properties": {
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "fields": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/field"
- }
- }
- },
- "required": [
- "id",
- "name",
- "fields"
- ],
- "additionalProperties": false
- },
- "field": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "alias": {
- "type": "string"
- },
- "display": {
- "type": "string"
- },
- "documentation": {
- "type": "string"
- },
- "anonymize": {
- "type": "boolean"
- }
- },
- "required": [
- "name",
- "display",
- "documentation"
- ],
- "additionalProperties": false
- }
- }
-}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json b/tests/artifacts/xdmod/schema/warehouse/export/DELETE-request.schema.json
similarity index 100%
rename from tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json
rename to tests/artifacts/xdmod/schema/warehouse/export/DELETE-request.schema.json
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse/export/DELETE-requests.schema.json
similarity index 100%
rename from tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json
rename to tests/artifacts/xdmod/schema/warehouse/export/DELETE-requests.schema.json
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse/export/GET-requests.schema.json
similarity index 100%
rename from tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json
rename to tests/artifacts/xdmod/schema/warehouse/export/GET-requests.schema.json
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json b/tests/artifacts/xdmod/schema/warehouse/export/POST-request.schema.json
similarity index 100%
rename from tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json
rename to tests/artifacts/xdmod/schema/warehouse/export/POST-request.schema.json
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json b/tests/artifacts/xdmod/schema/warehouse/export/error.schema.json
similarity index 100%
rename from tests/artifacts/xdmod/schema/warehouse-export/error.schema.json
rename to tests/artifacts/xdmod/schema/warehouse/export/error.schema.json
diff --git a/tests/artifacts/xdmod/user_controller/input/create_api_tokens.json b/tests/artifacts/xdmod/user_controller/input/create_api_tokens.json
deleted file mode 100644
index dab2e4a270..0000000000
--- a/tests/artifacts/xdmod/user_controller/input/create_api_tokens.json
+++ /dev/null
@@ -1,50 +0,0 @@
-[
- [
- {
- "user": "cd"
- }
- ],
- [
- {
- "user": "cs"
- }
- ],
- [
- {
- "user": "pi"
- }
- ],
- [
- {
- "user": "usr"
- }
- ],
- [
- {
- "user": "pub",
- "expected": {
- "api_get": {
- "http_code": 401,
- "schemas": {
- "success": "authentication_error.spec",
- "failure": "authentication_error.spec"
- }
- },
- "api_create": {
- "http_code": 401,
- "schemas": {
- "success": "authentication_error.spec",
- "failure": "authentication_error.spec"
- }
- },
- "api_revoke": {
- "http_code": 401,
- "schemas": {
- "success": "authentication_error.spec",
- "failure": "authentication_error.spec"
- }
- }
- }
- }
- ]
-]
diff --git a/tests/artifacts/xdmod/user_controller/input/create_api_tokens_defaults.json b/tests/artifacts/xdmod/user_controller/input/create_api_tokens_defaults.json
deleted file mode 100644
index 62d3a4bda9..0000000000
--- a/tests/artifacts/xdmod/user_controller/input/create_api_tokens_defaults.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "expected": {
- }
-}
diff --git a/tests/artifacts/xdmod/user_controller/input/test_endpoint_token_auth.json b/tests/artifacts/xdmod/user_controller/input/test_endpoint_token_auth.json
deleted file mode 100644
index ed561f8574..0000000000
--- a/tests/artifacts/xdmod/user_controller/input/test_endpoint_token_auth.json
+++ /dev/null
@@ -1,50 +0,0 @@
-[
- [
- {
- "user": "cd"
- }
- ],
- [
- {
- "user": "cs"
- }
- ],
- [
- {
- "user": "pi"
- }
- ],
- [
- {
- "user": "usr"
- }
- ],
- [
- {
- "user": "pub",
- "expected": {
- "api_get": {
- "http_code": [401, 400],
- "schemas": {
- "success": "authentication_error.spec",
- "failure": "authentication_error.spec"
- }
- },
- "api_create": {
- "http_code": [401, 400],
- "schemas": {
- "success": "authentication_error.spec",
- "failure": "authentication_error.spec"
- }
- },
- "api_revoke": {
- "http_code": [401, 400],
- "schemas": {
- "success": "authentication_error.spec",
- "failure": "authentication_error.spec"
- }
- }
- }
- }
- ]
-]
diff --git a/tests/artifacts/xdmod/user_controller/input/test_endpoint_token_auth_defaults.json b/tests/artifacts/xdmod/user_controller/input/test_endpoint_token_auth_defaults.json
deleted file mode 100644
index d0dc4dcd70..0000000000
--- a/tests/artifacts/xdmod/user_controller/input/test_endpoint_token_auth_defaults.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "test": {
- "description": "Metric Explorer get_dw_descripter",
- "url": "controllers/metric_explorer.php",
- "verb": "get",
- "parameters": {
- "operation": "get_dw_descripter"
- },
- "data": null
- },
- "expected": {
- "test": {
- "success": {
- "http_code": 200,
- "content_type": "application/json",
- "file_group": "user_controller",
- "file_name": "get_dw_descripter",
- "file_ext": ".json"
- },
- "failure": {
- "http_code": 401,
- "content_type": "application/json",
- "schema": "session_expired.spec"
- }
- }
- }
-}
diff --git a/tests/artifacts/xdmod/user_controller/input/test_rest_token_auth_defaults.json b/tests/artifacts/xdmod/user_controller/input/test_rest_token_auth_defaults.json
deleted file mode 100644
index 972134f3bc..0000000000
--- a/tests/artifacts/xdmod/user_controller/input/test_rest_token_auth_defaults.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "test": {
- "description": "Metric Explorer get_dw_descripter",
- "url": "rest/v1/warehouse/export/realms",
- "verb": "get",
- "parameters": null,
- "data": null
- },
- "expected": {
- "test": {
- "success": {
- "http_code": 200,
- "content_type": "application/json",
- "file_group": "user_controller",
- "file_name": "get_realms",
- "file_ext": ".json"
- },
- "failure": {
- "http_code": [401, 400],
- "content_type": "application/json",
- "schema": "authentication_error.spec"
- }
- }
- }
-}
diff --git a/tests/artifacts/xdmod/user_controller/input/token_auth_defaults.json b/tests/artifacts/xdmod/user_controller/input/token_auth_defaults.json
deleted file mode 100644
index 4be803eff4..0000000000
--- a/tests/artifacts/xdmod/user_controller/input/token_auth_defaults.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "expected": {
- "api_create": {
- "http_code": 200,
- "content_type": "application/json",
- "schemas": {
- "success": "create_user_token_success.spec",
- "failure": "create_user_token_failure.spec"
- }
- },
- "api_get": {
- "http_code": 200,
- "content_type": "application/json",
- "schemas": {
- "success": "get_user_token_success.spec",
- "failure": "get_user_token_failure.spec"
- }
- },
- "api_revoke": {
- "http_code": 200,
- "content_type": "application/json",
- "schemas": {
- "success": "revoke_user_token_success.spec",
- "failure": "revoke_user_token_failure.spec"
- }
- }
- }
-}
diff --git a/tests/artifacts/xdmod/user_controller/output/get_dw_descripter.json b/tests/artifacts/xdmod/user_controller/output/get_dw_descripter.json
deleted file mode 100644
index 88e41544ba..0000000000
--- a/tests/artifacts/xdmod/user_controller/output/get_dw_descripter.json
+++ /dev/null
@@ -1,408 +0,0 @@
-{
- "totalCount": 1,
- "data": [
- {
- "realms": {
- "Cloud": {
- "metrics": {
- "cloud_avg_cores_reserved": {
- "text": "Average Cores Reserved Weighted By Wall Hours",
- "info": "The average number of cores assigned to running sessions, weighted by wall hours.Core Hours<\/b>: The product of the number of cores assigned to a VM and its wall time, in hours.Wall Time:<\/b> The duration between the start and end times of an individual session.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.",
- "std_err": false
- },
- "cloud_avg_memory_reserved": {
- "text": "Average Memory Reserved Weighted By Wall Hours (Bytes)",
- "info": "The average amount of memory (in bytes) reserved by running sessions, weighted by wall hours.Wall Time:<\/b> The duration between the start and end times of an individual session.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.",
- "std_err": false
- },
- "cloud_avg_rv_storage_reserved": {
- "text": "Average Root Volume Storage Reserved Weighted By Wall Hours (Bytes)",
- "info": "The average amount of root volume storage space (in bytes) reserved by running sessions, weighted by wall hours.Wall Time:<\/b> The duration between the start and end times of an individual session.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.",
- "std_err": false
- },
- "cloud_core_utilization": {
- "text": "Core Hour Utilization (%)",
- "info": "A percentage that shows how many core hours were reserved over a time period against how many core hours a resource had available during that time period.Core Hours<\/b>: The product of the number of cores assigned to a VM and its wall time, in hours.Core Hours Available:<\/b> The total number of core hours available for a time period. Calculated by taking the product of the number of cores available over a time period, number of days in a time period and the number of hours in a day.",
- "std_err": false
- },
- "cloud_core_time": {
- "text": "Core Hours: Total",
- "info": "The total number of Core Hours consumed by running sessions.Core Hours<\/b>: The product of the number of cores assigned to a VM and its wall time, in hours.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.",
- "std_err": false
- },
- "cloud_num_sessions_running": {
- "text": "Number of Sessions Active",
- "info": "The total number of sessions on a cloud resource.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.Start:<\/b> A session start event is defined as the initial creation, resume from pause\/suspension, or unshelving of a VM. In the event that no such event has been collected, the first heartbeat event (e.g. a state report) is treated as the start of a new session.Stop:<\/b> A session stop event is defined as a pause, shelving, suspension, or termination event of a VM.",
- "std_err": false
- },
- "cloud_num_sessions_ended": {
- "text": "Number of Sessions Ended",
- "info": "The total number of sessions that were ended on a cloud resource. A session is ended when a VM is paused, shelved, stopped, or terminated on a cloud resource.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.Start:<\/b> A session start event is defined as the initial creation, resume from pause\/suspension, or unshelving of a VM. In the event that no such event has been collected, the first heartbeat event (e.g. a state report) is treated as the start of a new session.Stop:<\/b> A session stop event is defined as a pause, shelving, suspension, or termination event of a VM.",
- "std_err": false
- },
- "cloud_num_sessions_started": {
- "text": "Number of Sessions Started",
- "info": "The total number of sessions started on a cloud resource. A session begins when a VM is created, unshelved, or resumes running on a cloud resource.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.Start:<\/b> A session start event is defined as the initial creation, resume from pause\/suspension, or unshelving of a VM. In the event that no such event has been collected, the first heartbeat event (e.g. a state report) is treated as the start of a new session.Stop:<\/b> A session stop event is defined as a pause, shelving, suspension, or termination event of a VM.",
- "std_err": false
- },
- "cloud_avg_wallduration_hours": {
- "text": "Wall Hours: Per Session",
- "info": "The total number of sessions on a cloud resource.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.Start:<\/b> A session start event is defined as the initial creation, resume from pause\/suspension, or unshelving of a VM. In the event that no such event has been collected, the first heartbeat event (e.g. a state report) is treated as the start of a new session.Stop:<\/b> A session stop event is defined as a pause, shelving, suspension, or termination event of a VM.",
- "std_err": false
- },
- "cloud_wall_time": {
- "text": "Wall Hours: Total",
- "info": "The average number of cores assigned to running sessions, weighted by wall hours.Core Hours<\/b>: The product of the number of cores assigned to a VM and its wall time, in hours.Wall Time:<\/b> The duration between the start and end times of an individual session.Session:<\/b> A session is defined as a discrete run of a virtual machine (VM) on a cloud resource; i.e. any start and stop of a VM. For example, if a single VM is stopped and restarted ten times in a given day, this would be counted as ten sessions for that day.",
- "std_err": false
- }
- },
- "dimensions": {
- "none": {
- "text": "None",
- "info": "Summarizes Cloud data reported to the Screwdriver database."
- },
- "nsfdirectorate": {
- "text": "Decanal Unit",
- "info": "Decanal Unit"
- },
- "parentscience": {
- "text": "Department",
- "info": "Department"
- },
- "domain": {
- "text": "Domain",
- "info": "A domain is a high-level container for projects, users and groups"
- },
- "configuration": {
- "text": "Instance Type",
- "info": "The instance type of a virtual machine."
- },
- "pi": {
- "text": "PI",
- "info": "The principal investigator of a project."
- },
- "fieldofscience": {
- "text": "PI Group",
- "info": "PI Group"
- },
- "project": {
- "text": "Project",
- "info": "The project associated with a running session of a virtual machine."
- },
- "resource": {
- "text": "Resource",
- "info": "A resource is defined as any infrastructure that hosts virtual machines."
- },
- "submission_venue": {
- "text": "Submission Venue",
- "info": "The venue that a job or cloud instance was initiated from."
- },
- "username": {
- "text": "System Username",
- "info": "The specific system username of the users who ran jobs."
- },
- "person": {
- "text": "User",
- "info": "A person on a principal investigator's allocation, able to spin up and manipulate VM instances."
- },
- "vm_size": {
- "text": "VM Size: Cores",
- "info": "A categorization of sessions into discrete groups based on the number of cores used by each VM."
- },
- "vm_size_memory": {
- "text": "VM Size: Memory",
- "info": "A categorization of sessions into discrete groups based on the amount of memory reserved by each VM."
- }
- },
- "text": "Cloud",
- "category": "Cloud"
- },
- "Jobs": {
- "metrics": {
- "avg_cpu_hours": {
- "text": "CPU Hours: Per Job",
- "info": "The average CPU hours (number of CPU cores x wall time hours) per Screwdriver job.For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours.",
- "std_err": true
- },
- "total_cpu_hours": {
- "text": "CPU Hours: Total",
- "info": "The total CPU hours (number of CPU cores x wall time hours) used by Screwdriver jobs.For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours.",
- "std_err": false
- },
- "avg_gpus": {
- "text": "GPU Count: Per Job",
- "info": "The average job size per Screwdriver job. Job Size: <\/i>The number of GPUs used by a (parallel) job.",
- "std_err": true
- },
- "avg_gpu_hours": {
- "text": "GPU Hours: Per Job",
- "info": "The average GPU hours (number of GPU cores x wall time hours) per Screwdriver job.For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours.",
- "std_err": false
- },
- "total_gpu_hours": {
- "text": "GPU Hours: Total",
- "info": "The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours.",
- "std_err": false
- },
- "max_processors": {
- "text": "Job Size: Max (Core Count)",
- "info": "The maximum size Screwdriver job in number of cores.Job Size: <\/i>The total number of processor cores used by a (parallel) job.",
- "std_err": false
- },
- "min_processors": {
- "text": "Job Size: Min (Core Count)",
- "info": "The minimum size Screwdriver job in number of cores.Job Size: <\/i>The total number of processor cores used by a (parallel) job.",
- "std_err": false
- },
- "normalized_avg_processors": {
- "text": "Job Size: Normalized (% of Total Cores)",
- "info": "The percentage average size Screwdriver job over total machine cores. Normalized Job Size: <\/i>The percentage total number of processor cores used by a (parallel) job over the total number of cores on the machine.",
- "std_err": false
- },
- "avg_processors": {
- "text": "Job Size: Per Job (Core Count)",
- "info": "The average job size per Screwdriver job. Job Size: <\/i>The number of processor cores used by a (parallel) job.",
- "std_err": true
- },
- "avg_job_size_weighted_by_cpu_hours": {
- "text": "Job Size: Weighted By CPU Hours (Core Count)",
- "info": "The average Screwdriver job size weighted by CPU Hours. Defined as Average Job Size Weighted By CPU Hours: <\/i> sum(i = 0 to n){ job i core count * job i cpu hours}\/sum(i = 0 to n){job i cpu hours}",
- "std_err": false
- },
- "avg_job_size_weighted_by_gpu_hours": {
- "text": "Job Size: Weighted By GPU Hours (GPU Count)",
- "info": "The average Screwdriver job size weighted by GPU Hours. Defined as Average Job Size Weighted By GPU Hours: <\/i> sum(i = 0 to n){ job i core count * job i gpu hours}\/sum(i = 0 to n){job i gpu hours}",
- "std_err": false
- },
- "avg_node_hours": {
- "text": "Node Hours: Per Job",
- "info": "The average node hours (number of nodes x wall time hours) per Screwdriver job.",
- "std_err": true
- },
- "total_node_hours": {
- "text": "Node Hours: Total",
- "info": "The total node hours (number of nodes x wall time hours) used by Screwdriver jobs.",
- "std_err": false
- },
- "job_count": {
- "text": "Number of Jobs Ended",
- "info": "The total number of Screwdriver jobs that ended within the selected duration.",
- "std_err": false
- },
- "running_job_count": {
- "text": "Number of Jobs Running",
- "info": "The total number of Screwdriver jobs that are running.",
- "std_err": false
- },
- "started_job_count": {
- "text": "Number of Jobs Started",
- "info": "The total number of Screwdriver jobs that started executing within the selected duration.",
- "std_err": false
- },
- "submitted_job_count": {
- "text": "Number of Jobs Submitted",
- "info": "The total number of Screwdriver jobs that submitted\/queued within the selected duration.",
- "std_err": false
- },
- "active_pi_count": {
- "text": "Number of PIs: Active",
- "info": "The total number of PIs that used Screwdriver resources.",
- "std_err": false
- },
- "active_resource_count": {
- "text": "Number of Resources: Active",
- "info": "The total number of active Screwdriver resources.",
- "std_err": false
- },
- "active_person_count": {
- "text": "Number of Users: Active",
- "info": "The total number of users that used Screwdriver resources.",
- "std_err": false
- },
- "utilization": {
- "text": "Screwdriver Utilization (%)",
- "info": "The percentage of core time that a resource has been running jobs. Screwdriver Utilization:<\/i> The ratio of the total CPU hours consumed by jobs over a given time period divided by the maximum CPU hours that the system could deliver (based on the number of cores present on the resources). This value does not take into account downtimes or outages. It is just calculated based on the number of cores in the resource specifications.",
- "std_err": false
- },
- "expansion_factor": {
- "text": "User Expansion Factor",
- "info": "The average job size per Screwdriver job. Job Size: <\/i>The number of processor cores used by a (parallel) job.",
- "std_err": false
- },
- "avg_waitduration_hours": {
- "text": "Wait Hours: Per Job",
- "info": "The average time, in hours, a Screwdriver job waits before execution on the designated resource.Wait Time: <\/i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute.",
- "std_err": true
- },
- "total_waitduration_hours": {
- "text": "Wait Hours: Total",
- "info": "The total time, in hours, Screwdriver jobs waited before execution on their designated resource.Wait Time: <\/i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute.",
- "std_err": false
- },
- "avg_wallduration_hours": {
- "text": "Wall Hours: Per Job",
- "info": "The average time, in hours, a job takes to execute.In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation. Wall Time:<\/i> Wall time is defined as the linear time between start and end time of execution for a particular job.",
- "std_err": true
- },
- "total_wallduration_hours": {
- "text": "Wall Hours: Total",
- "info": "The total time, in hours, Screwdriver jobs took to execute.Wall Time:<\/i> Wall time is defined as the linear time between start and end time of execution for a particular job.",
- "std_err": false
- }
- },
- "dimensions": {
- "none": {
- "text": "None",
- "info": "Summarizes Jobs data reported to the Screwdriver database."
- },
- "nsfdirectorate": {
- "text": "Decanal Unit",
- "info": "Decanal Unit"
- },
- "parentscience": {
- "text": "Department",
- "info": "Department"
- },
- "gpucount": {
- "text": "GPU Count",
- "info": "A categorization of jobs into discrete groups based on the number of GPUs used by each job."
- },
- "jobsize": {
- "text": "Job Size",
- "info": "A categorization of jobs into discrete groups based on the number of cores used by each job."
- },
- "jobwaittime": {
- "text": "Job Wait Time",
- "info": "A categorization of jobs into discrete groups based on the total linear time each job waited."
- },
- "jobwalltime": {
- "text": "Job Wall Time",
- "info": "A categorization of jobs into discrete groups based on the total linear time each job took to execute."
- },
- "nodecount": {
- "text": "Node Count",
- "info": "A categorization of jobs into discrete groups based on node count."
- },
- "pi": {
- "text": "PI",
- "info": "The principal investigator of a project."
- },
- "fieldofscience": {
- "text": "PI Group",
- "info": "PI Group"
- },
- "qos": {
- "text": "QOS",
- "info": "The quality of service of the job"
- },
- "queue": {
- "text": "Queue",
- "info": "Queue pertains to the low level job queues on each resource."
- },
- "resource": {
- "text": "Resource",
- "info": "A resource is a remote computer that can run jobs."
- },
- "resource_type": {
- "text": "Resource Type",
- "info": "A categorization of resources into by their general capabilities."
- },
- "username": {
- "text": "System Username",
- "info": "The specific system username of the users who ran jobs."
- },
- "person": {
- "text": "User",
- "info": "A person who is on a PIs allocation, hence able run jobs on resources."
- }
- },
- "text": "Jobs",
- "category": "Jobs"
- },
- "Storage": {
- "metrics": {
- "avg_file_count": {
- "text": "File Count (Number of files)",
- "info": "Average number of files.",
- "std_err": false
- },
- "avg_logical_usage": {
- "text": "Logical Usage (Bytes)",
- "info": "Average logical file system usage measured in bytes.",
- "std_err": false
- },
- "avg_physical_usage": {
- "text": "Physical Usage (Bytes)",
- "info": "Average physical file system usage measured in bytes.",
- "std_err": false
- },
- "avg_logical_utilization": {
- "text": "Quota Utilization: Logical (%)",
- "info": "Average logical file system usage as a percentage of the quota soft threshold.",
- "std_err": false
- },
- "avg_hard_threshold": {
- "text": "Quota: Hard Threshold (Bytes)",
- "info": "Average file system quota hard threshold measured in bytes.",
- "std_err": false
- },
- "avg_soft_threshold": {
- "text": "Quota: Soft Threshold (Bytes)",
- "info": "Average file system quota soft threshold measured in bytes.",
- "std_err": false
- },
- "user_count": {
- "text": "User Count (Number of Users)",
- "info": "Number of Screwdriver users with file system usage data.",
- "std_err": false
- }
- },
- "dimensions": {
- "none": {
- "text": "None",
- "info": "Summarizes Storage data reported to the Screwdriver database."
- },
- "nsfdirectorate": {
- "text": "Decanal Unit",
- "info": "Decanal Unit"
- },
- "parentscience": {
- "text": "Department",
- "info": "Department"
- },
- "mountpoint": {
- "text": "Mountpoint",
- "info": "Storage by Mountpoint"
- },
- "pi": {
- "text": "PI",
- "info": "The principal investigator of a project."
- },
- "fieldofscience": {
- "text": "PI Group",
- "info": "PI Group"
- },
- "resource": {
- "text": "Resource",
- "info": "A resource is a remote computer that can run jobs."
- },
- "resource_type": {
- "text": "Resource Type",
- "info": "A categorization of resources into by their general capabilities."
- },
- "username": {
- "text": "System Username",
- "info": "The specific system username of the users who ran jobs."
- },
- "person": {
- "text": "User",
- "info": "A person who is on a PIs allocation, hence able run jobs on resources."
- }
- },
- "text": "Storage",
- "category": "Storage"
- }
- }
- }
- ]
-}
diff --git a/tests/artifacts/xdmod/user_controller/output/get_realms.json b/tests/artifacts/xdmod/user_controller/output/get_realms.json
deleted file mode 100644
index b2b0e88f40..0000000000
--- a/tests/artifacts/xdmod/user_controller/output/get_realms.json
+++ /dev/null
@@ -1,326 +0,0 @@
-{
- "success": true,
- "data": [
- {
- "id": "Jobs",
- "name": "Jobs",
- "fields": [
- {
- "name": "Local Job Id",
- "alias": "Local Job Id",
- "display": "Local Job Id",
- "anonymize": false,
- "documentation": "The unique identifier assigned to the job by the job scheduler."
- },
- {
- "name": "Resource",
- "alias": "Resource",
- "display": "Resource",
- "anonymize": false,
- "documentation": "The resource that ran the job."
- },
- {
- "name": "Timezone",
- "alias": "Timezone",
- "display": "Timezone",
- "anonymize": false,
- "documentation": "The timezone of the resource."
- },
- {
- "name": "System Username",
- "alias": "System Username",
- "display": "System Username (Deidentified)",
- "anonymize": true,
- "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource."
- },
- {
- "name": "User",
- "alias": "User",
- "display": "User",
- "anonymize": false,
- "documentation": "The name of the job owner."
- },
- {
- "name": "Organization",
- "alias": "Organization",
- "display": "Organization",
- "anonymize": false,
- "documentation": "The organization of the person who ran the task"
- },
- {
- "name": "Quality of Service",
- "alias": "Quality of Service",
- "display": "Quality of Service",
- "anonymize": false,
- "documentation": "The job quality of service."
- },
- {
- "name": "Submit Time",
- "alias": "Submit Time",
- "display": "Submit Time (Timestamp)",
- "anonymize": false,
- "documentation": "Task submission time"
- },
- {
- "name": "Start Time",
- "alias": "Start Time",
- "display": "Start Time (Timestamp)",
- "anonymize": false,
- "documentation": "The time that the job started running."
- },
- {
- "name": "End Time",
- "alias": "End Time",
- "display": "End Time (Timestamp)",
- "anonymize": false,
- "documentation": "The time that the job ended."
- },
- {
- "name": "Eligible Time",
- "alias": "Eligible Time",
- "display": "Eligible Time (Timestamp)",
- "anonymize": false,
- "documentation": "The time that the job was eligible for scheduling by the resource manager."
- },
- {
- "name": "Nodes",
- "alias": "Nodes",
- "display": "Nodes",
- "anonymize": false,
- "documentation": "The number of nodes that were assigned to the job."
- },
- {
- "name": "Cores",
- "alias": "Cores",
- "display": "Cores",
- "anonymize": false,
- "documentation": "The number of cores that were assigned to the job."
- },
- {
- "name": "GPUs",
- "alias": "GPUs",
- "display": "GPUs",
- "anonymize": false,
- "documentation": "The number of GPUs that were assigned to the job."
- },
- {
- "name": "Memory Used",
- "alias": "Memory Used",
- "display": "Memory Used",
- "anonymize": false,
- "documentation": "Memory consumed as reported by the resource manager."
- },
- {
- "name": "Wall Time",
- "alias": "Wall Time",
- "display": "Wall Time",
- "anonymize": false,
- "documentation": "Overall job duration."
- },
- {
- "name": "Wait Time",
- "alias": "Wait Time",
- "display": "Wait Time",
- "anonymize": false,
- "documentation": "Time the job waited in the queue"
- },
- {
- "name": "Core Time",
- "alias": "Core Time",
- "display": "Core Time",
- "anonymize": false,
- "documentation": "The amount of CPU core time (Core Count * Wall Time)"
- },
- {
- "name": "GPU Time",
- "alias": "GPU Time",
- "display": "GPU Time",
- "anonymize": false,
- "documentation": "The amount of GPU time (GPU Count * Wall Time)"
- },
- {
- "name": "Exit Code",
- "alias": "Exit Code",
- "display": "Exit Code",
- "anonymize": false,
- "documentation": "The code that the job exited with."
- },
- {
- "name": "Exit State",
- "alias": "Exit State",
- "display": "Exit State",
- "anonymize": false,
- "documentation": "The state of the job when it completed."
- },
- {
- "name": "Requested Cores",
- "alias": "Requested Cores",
- "display": "Requested Cores",
- "anonymize": false,
- "documentation": "The number of CPUs required by the job."
- },
- {
- "name": "Requested memory",
- "alias": "Requested memory",
- "display": "Requested memory",
- "anonymize": false,
- "documentation": "The amount of memory required by the job."
- },
- {
- "name": "Requested Wall Time",
- "alias": "Requested Wall Time",
- "display": "Requested Wall Time",
- "anonymize": false,
- "documentation": "The time limit of the job."
- },
- {
- "name": "Queue",
- "alias": "Queue",
- "display": "Queue",
- "anonymize": false,
- "documentation": "The name of the queue to which the job was submitted."
- },
- {
- "name": "Decanal Unit",
- "alias": "Decanal Unit",
- "display": "Decanal Unit",
- "anonymize": false,
- "documentation": "Decanal Unit"
- },
- {
- "name": "Department",
- "alias": "Department",
- "display": "Department",
- "anonymize": false,
- "documentation": "Department"
- },
- {
- "name": "PI Group",
- "alias": "PI Group",
- "display": "PI Group",
- "anonymize": false,
- "documentation": "PI Group"
- }
- ]
- },
- {
- "id": "Cloud",
- "name": "Cloud",
- "fields": [
- {
- "name": "Instance ID",
- "alias": "Instance ID",
- "display": "Instance ID",
- "anonymize": false,
- "documentation": "The UUID of the VM"
- },
- {
- "name": "Resource",
- "alias": "Resource",
- "display": "Resource",
- "anonymize": false,
- "documentation": "The resource that ran the job."
- },
- {
- "name": "System Username",
- "alias": "System Username",
- "display": "System Username (Deidentified)",
- "anonymize": true,
- "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource."
- },
- {
- "name": "User",
- "alias": "User",
- "display": "User",
- "anonymize": false,
- "documentation": "The name of the job owner."
- },
- {
- "name": "Decanal Unit",
- "alias": "Decanal Unit",
- "display": "Decanal Unit",
- "anonymize": false,
- "documentation": "Decanal Unit"
- },
- {
- "name": "Department",
- "alias": "Department",
- "display": "Department",
- "anonymize": false,
- "documentation": "Department"
- },
- {
- "name": "PI Group",
- "alias": "PI Group",
- "display": "PI Group",
- "anonymize": false,
- "documentation": "PI Group"
- },
- {
- "name": "Project",
- "alias": "Project",
- "display": "Project",
- "anonymize": false,
- "documentation": "The name of the project the VM is assigned to."
- },
- {
- "name": "Principal Investigator",
- "alias": "Principal Investigator",
- "display": "Principal Investigator",
- "anonymize": false,
- "documentation": "The name of the PI assigned to the VM."
- },
- {
- "name": "Instance Type",
- "alias": "Instance Type",
- "display": "Instance Type",
- "anonymize": false,
- "documentation": "The instance type for the VM."
- },
- {
- "name": "Number of Cores",
- "alias": "Number of Cores",
- "display": "Number of Cores",
- "anonymize": false,
- "documentation": "The amount of cores a VM has."
- },
- {
- "name": "Memory(MB)",
- "alias": "Memory(MB)",
- "display": "Memory(MB)",
- "anonymize": false,
- "documentation": "The amount of memory a VM has in MB."
- },
- {
- "name": "Root Volume Size",
- "alias": "Root Volume Size",
- "display": "Root Volume Size",
- "anonymize": false,
- "documentation": "The size of the root volume for the VM."
- },
- {
- "name": "Domain",
- "alias": "Domain",
- "display": "Domain",
- "anonymize": false,
- "documentation": "Domain the VM was created in. Applicable to Open Stack VM\u0027s only."
- },
- {
- "name": "Host",
- "alias": "Host",
- "display": "Host",
- "anonymize": false,
- "documentation": "The name of the host the VM resides on."
- },
- {
- "name": "Start Date",
- "alias": "Start Date",
- "display": "Start Date",
- "anonymize": false,
- "documentation": "The start date of a VM."
- }
- ]
- }
- ],
- "total": 2
-}
diff --git a/tests/artifacts/xdmod/user_controller/output/token_unauthorized.json b/tests/artifacts/xdmod/user_controller/output/token_unauthorized.json
deleted file mode 100644
index a173297482..0000000000
--- a/tests/artifacts/xdmod/user_controller/output/token_unauthorized.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "success": false,
- "message": "An API token must be supplied to use this endpoint."
-}
diff --git a/tests/integration/lib/BaseTest.php b/tests/integration/lib/BaseTest.php
index 607f10b3d9..203c34ea72 100644
--- a/tests/integration/lib/BaseTest.php
+++ b/tests/integration/lib/BaseTest.php
@@ -2,41 +2,216 @@
namespace IntegrationTests;
-use JsonSchema\Validator;
-use \TestHarness\Utilities;
+use CCR\Json;
+use Exception;
+use Swaggest\JsonSchema\Schema;
+use TestHarness\Utilities;
+use TestHarness\TestFiles;
abstract class BaseTest extends \PHPUnit_Framework_TestCase
{
protected static $XDMOD_REALMS;
+ protected static $testFiles;
public static function setUpBeforeClass()
{
self::$XDMOD_REALMS = Utilities::getRealmsToTest();
}
- public static function getRealms()
+ protected static function getRealms()
{
return Utilities::getRealmsToTest();
}
- /**
- * Validate the provided $json w/ provided Json Schema $schemaObject and asserting that there were no errors. If the
- * validation is successful then the decoded value is returned.
- *
- * @param mixed $json the JSON structure to be validated.
- * @param \stdClass $schemaObject the JsonSchema object to be used to validate $json.
- * @return mixed the decoded, valid json structure.
- */
- protected function validateJson($json, \stdClass $schemaObject)
+ public static function getTestFiles()
{
- $validator = new Validator();
- $actualDecoded = json_decode(json_encode($json));
- $validator->validate($actualDecoded, $schemaObject);
- $errors = array();
- foreach ($validator->getErrors() as $err) {
- $errors[] = sprintf("[%s] %s\n", $err['property'], $err['message']);
- }
- $this->assertEmpty($errors, implode("\n", $errors) . "\n" . json_encode($json, JSON_PRETTY_PRINT));
- return $actualDecoded;
+ if (!isset(self::$testFiles)) {
+ self::$testFiles = new TestFiles(__DIR__ . '/../../');
+ }
+ return self::$testFiles;
+ }
+
+ public function provideBaseRoles()
+ {
+ return array(
+ array('pub'),
+ array('cd'),
+ array('cs'),
+ array('pi'),
+ array('usr'),
+ array('mgr')
+ );
+ }
+
+ public function makeRequest(
+ $testHelper,
+ $path,
+ $verb,
+ $params = null,
+ $data = null,
+ $expectedHttpCode = null,
+ $expectedContentType = null,
+ $expectedFileGroup = null,
+ $expectedFileName = null,
+ $validationType = null,
+ $expectedHeaders = null
+ ) {
+ $response = null;
+ switch ($verb) {
+ case 'get':
+ $response = $testHelper->$verb($path, $params);
+ break;
+ case 'post':
+ case 'delete':
+ case 'patch':
+ $response = $testHelper->$verb($path, $params, $data);
+ break;
+ }
+ $actualHttpCode = isset($response) ? $response[1]['http_code'] : null;
+ $actualContentType = isset($response) ? $response[1]['content_type'] : null;
+ $actualResponseBody = isset($response) ? $response[0] : array();
+ $message = "PATH: $path\nVERB: $verb\nHEADERS: "
+ . json_encode($testHelper->getheaders(), JSON_PRETTY_PRINT)
+ . "\nPARAMS: " . json_encode($params, JSON_PRETTY_PRINT)
+ . "\nDATA: " . json_encode($data, JSON_PRETTY_PRINT)
+ . "\nEXPECTED HTTP CODE: $expectedHttpCode"
+ . "\nACTUAL HTTP CODE: $actualHttpCode"
+ . "\nEXPECTED CONTENT TYPE: $expectedContentType"
+ . "\nACTUAL CONTENT TYPE: $actualContentType";
+ if (isset($expectedHttpCode)) {
+ $this->assertSame(
+ $expectedHttpCode,
+ $actualHttpCode,
+ $message
+ );
+ }
+ if (isset($expectedContentType)) {
+ $this->assertSame(
+ $expectedContentType,
+ $actualContentType,
+ $message
+ );
+ }
+
+ $actual = json_decode(json_encode($actualResponseBody));
+
+ if (isset($expectedFileName)) {
+ $this->validateJson(
+ $actual,
+ $expectedFileGroup,
+ $expectedFileName,
+ 'output',
+ $validationType,
+ $message
+ );
+ }
+
+ if (isset($expectedHeaders)) {
+ foreach ($expectedHeaders as $key => $value) {
+ $this->assertArrayHasKey($key, $response[2], $message);
+ $this->assertSame(
+ $value,
+ trim($response[2][$key]),
+ $message
+ );
+ }
+ }
+
+ return $actual;
+ }
+
+ public function validateJson(
+ $json,
+ $testGroup,
+ $fileName,
+ $fileType = 'output',
+ $validationType = 'schema',
+ $message = ''
+ ) {
+ $expectedFile = self::getTestFiles()->getFile(
+ $testGroup,
+ $fileName,
+ $fileType,
+ '.json'
+ );
+ $actualObject = json_decode(json_encode($json), false);
+ if ('exact' === $validationType) {
+ $expectedObject = self::loadRawJsonFile(
+ $expectedFile,
+ $validationType
+ );
+ $this->assertSame(
+ json_encode($expectedObject),
+ json_encode($actualObject),
+ $message . "\nEXPECTED OUTPUT FILE: $expectedFile"
+ );
+ } elseif ('schema' === $validationType) {
+ $expectedObject = Json::loadFile($expectedFile, false);
+ $expectedObject = self::resolveRemoteSchemaRefs(
+ $expectedObject,
+ dirname($expectedFile)
+ );
+ $schema = Schema::import($expectedObject);
+ try {
+ $schema->in($actualObject);
+ } catch (Exception $e) {
+ $a = json_encode($actualObject);
+ $this->fail(
+ $e->getMessage() . "\nEXPECTED SCHEMA: $expectedFile"
+ . "\nACTUAL OBJECT: "
+ . (strlen($a) > 1000 ? substr($a, 0, 1000) . '...' : $a)
+ );
+ }
+ }
+ return $actualObject;
+ }
+
+ private function loadRawJsonFile($file, $validationType) {
+ $object = Json::loadFile($file, true);
+ if ('exact' === $validationType) {
+ if (isset($object['$extends'])) {
+ $parentObject = self::loadRawJsonFile(
+ self::resolveExternalFilePath(
+ dirname($file),
+ $object['$extends']
+ ),
+ $validationType
+ );
+ $object = array_replace_recursive($parentObject, $object);
+ unset($object['$extends']);
+ }
+ }
+ return $object;
+ }
+
+ private static function resolveRemoteSchemaRefs($obj, $schemaDir)
+ {
+ foreach ($obj as $key => $value) {
+ if ('$ref' === $key && '#' !== $value[0]) {
+ $obj->$key = self::resolveExternalFilePath($schemaDir, $value);
+ } elseif ('object' === gettype($value)
+ || 'array' === gettype($value)) {
+ $value = self::resolveRemoteSchemaRefs($value, $schemaDir);
+ }
+ }
+ return $obj;
+ }
+
+ private static function resolveExternalFilePath($parentPath, $path)
+ {
+ if (false !== strpos($path, '${INTEGRATION_ROOT}')) {
+ return self::getTestFiles()->getFile(
+ 'integration',
+ str_replace(
+ '${INTEGRATION_ROOT}/',
+ '',
+ $path
+ ),
+ '',
+ ''
+ );
+ } else {
+ return $parentPath . '/' . $path;
+ }
}
}
diff --git a/tests/integration/lib/Controllers/BaseUserAdminTest.php b/tests/integration/lib/Controllers/BaseUserAdminTest.php
index cd7dd6b688..94c2120ae6 100644
--- a/tests/integration/lib/Controllers/BaseUserAdminTest.php
+++ b/tests/integration/lib/Controllers/BaseUserAdminTest.php
@@ -4,7 +4,6 @@
use CCR\Json;
use Exception;
-use TestHarness\TestFiles;
use TestHarness\XdmodTestHelper;
use TestHarness\PeopleHelper;
use IntegrationTests\BaseTest;
@@ -47,11 +46,6 @@ abstract class BaseUserAdminTest extends BaseTest
*/
protected static $existingUsers = array();
- /**
- * @var TestFiles
- */
- protected $testFiles;
-
/**
* @var PeopleHelper
*/
@@ -60,18 +54,9 @@ abstract class BaseUserAdminTest extends BaseTest
protected function setUp()
{
$this->helper = new XdmodTestHelper();
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
$this->peopleHelper = new PeopleHelper();
}
- public function getTestFiles()
- {
- if (!isset($this->testFiles)) {
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
- }
- return $this->testFiles;
- }
-
public static function tearDownAfterClass()
{
foreach (self::$newUsers as $username => $userId) {
@@ -239,7 +224,7 @@ protected function createUser(array $options)
// retrieve the expected results of submitting the 'create_user' request
// with the supplied arguments.
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile(
+ parent::getTestFiles()->getFile(
'user_admin',
$output
)
@@ -322,7 +307,7 @@ protected function updateCurrentUser($userId, $password = null, $firstName = nul
);
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'test.update_user')
+ parent::getTestFiles()->getFile('user_admin', 'test.update_user')
);
$this->validateResponse($updateUserResponse);
diff --git a/tests/integration/lib/Controllers/ControllerTest.php b/tests/integration/lib/Controllers/ControllerTest.php
index f49b63c71b..2665fc4a25 100644
--- a/tests/integration/lib/Controllers/ControllerTest.php
+++ b/tests/integration/lib/Controllers/ControllerTest.php
@@ -1,11 +1,11 @@
testFiles)) {
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
- }
- return $this->testFiles;
- }
-
protected function setUp()
{
$this->helper = new XdmodTestHelper(__DIR__ . '/../../../');
@@ -64,7 +50,7 @@ public function testEnumExistingUsers()
);
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('controllers', 'enum_existing_users')
+ parent::getTestFiles()->getFile('controllers', 'enum_existing_users')
);
$this->assertEquals($expected['success'], $actual['success']);
@@ -83,7 +69,7 @@ public function testEnumExistingUsers()
public function testEnumUserTypes()
{
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('controllers', 'enum_user_types-8.0.0')
+ parent::getTestFiles()->getFile('controllers', 'enum_user_types-8.0.0')
);
$this->helper->authenticateDashboard('mgr');
@@ -115,7 +101,7 @@ public function testEnumUserTypes()
public function testEnumRoles()
{
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('controllers', 'enum_roles-add_default_center')
+ parent::getTestFiles()->getFile('controllers', 'enum_roles-add_default_center')
);
$this->helper->authenticateDashboard('mgr');
@@ -200,7 +186,7 @@ public function testListUsers(array $options)
$data = $response[0];
$expected = Json::loadFile(
- $this->getTestFiles()->getFile('controllers', $outputFile)
+ parent::getTestFiles()->getFile('controllers', $outputFile)
);
// Retrieve the users value and ensure that it is sorted in the correct order.
$actualUsers = $data['users'];
@@ -216,14 +202,14 @@ public function testListUsers(array $options)
public function listUsersGroupProvider()
{
return Json::loadFile(
- $this->getTestFiles()->getFile('controllers', 'list_users', 'input')
+ parent::getTestFiles()->getFile('controllers', 'list_users', 'input')
);
}
public function testEnumUserTypesAndRoles()
{
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('controllers', 'enum_user_types_and_roles-update_enum_user_types_and_roles')
+ parent::getTestFiles()->getFile('controllers', 'enum_user_types_and_roles-update_enum_user_types_and_roles')
);
$this->helper->authenticateDashboard('mgr');
@@ -287,7 +273,7 @@ public function testSabUserEnumTgUsers()
$this->assertEquals('success', $data['message'], "Expected the 'message' property to equal 'success'. Received: " . $data['message']);
$this->assertCount(300, $data['users'], "Expected 300 users to be returned. Received: " . count($data['users']));
- $expectedFilePath = $this->getTestFiles()->getFile('controllers', 'sab_user_enum_tg_users');
+ $expectedFilePath = parent::getTestFiles()->getFile('controllers', 'sab_user_enum_tg_users');
if (!is_file($expectedFilePath)) {
file_put_contents($expectedFilePath, json_encode($data, JSON_PRETTY_PRINT) . "\n");
@@ -482,7 +468,7 @@ public function testEnumTargetAddresses(array $options)
$expected = $options['expected'];
$expectedFile = $expected['file'];
- $expectedFileName = $this->getTestFiles()->getFile('controllers', $expectedFile);
+ $expectedFileName = parent::getTestFiles()->getFile('controllers', $expectedFile);
$expectedContentType = array_key_exists('content_type', $expected) ? $expected['content_type'] : 'text/html; charset=UTF-8';
$expectedHttpCode = array_key_exists('http_code', $expected) ? $expected['http_code'] : 200;
@@ -526,7 +512,7 @@ public function testEnumTargetAddresses(array $options)
*/
public function provideEnumTargetAddresses()
{
- $data = JSON::loadFile($this->getTestFiles()->getFile('controllers', 'enum_target_addresses-update_enum_user_types_and_roles', 'input'));
+ $data = JSON::loadFile(parent::getTestFiles()->getFile('controllers', 'enum_target_addresses-update_enum_user_types_and_roles', 'input'));
$helper = new XdmodTestHelper();
$helper->authenticateDashboard('mgr');
diff --git a/tests/integration/lib/Controllers/MetricExplorerTest.php b/tests/integration/lib/Controllers/MetricExplorerTest.php
index 97a46281ca..adf2f061f9 100644
--- a/tests/integration/lib/Controllers/MetricExplorerTest.php
+++ b/tests/integration/lib/Controllers/MetricExplorerTest.php
@@ -3,6 +3,7 @@
namespace IntegrationTests\Controllers;
use IntegrationTests\BaseTest;
+use TestHarness\TokenHelper;
class MetricExplorerTest extends BaseTest
{
@@ -12,46 +13,34 @@ protected function setUp()
}
/**
- * Checks the structure of the DwDescripter response.
+ * @dataProvider provideBaseRoles
*/
- public function testGetDwDescripter()
+ public function testGetDwDescripter($role)
{
- $this->helper->authenticate('cd');
-
- $response = $this->helper->post('/controllers/metric_explorer.php', null, array('operation' => 'get_dw_descripter'));
-
- $this->assertEquals('application/json', $response[1]['content_type']);
- $this->assertEquals(200, $response[1]['http_code']);
-
-
- $dwdata = $response[0];
-
- $this->assertArrayHasKey('totalCount', $dwdata);
- $this->assertArrayHasKey('data', $dwdata);
- $this->assertEquals($dwdata['totalCount'], count($dwdata['data']));
-
- foreach($dwdata['data'] as $entry)
- {
- $this->assertArrayHasKey('realms', $entry);
- foreach($entry['realms'] as $realm)
- {
- $this->assertArrayHasKey('dimensions', $realm);
- $this->assertArrayHasKey('metrics', $realm);
+ $tokenHelper = new TokenHelper(
+ $this,
+ $this->helper,
+ $role,
+ 'controllers/metric_explorer.php',
+ 'post',
+ null,
+ array(
+ 'operation' => 'get_dw_descripter'
+ ),
+ 'controller',
+ 'token_optional'
+ );
+ $tokenHelper->runEndpointTests(
+ function ($token) use ($tokenHelper) {
+ $tokenHelper->runEndpointTest(
+ $token,
+ 'get_dw_descripter.spec',
+ 200,
+ 'integration/controllers/metric_explorer',
+ 'schema'
+ );
}
- }
- }
-
- /**
- * checks that you need to be authenticated to get_dw_descripter
- */
- public function testGetDwDescripterNoAuth()
- {
- // note - not authenticated
-
- $response = $this->helper->post('/controllers/metric_explorer.php', null, array('operation' => 'get_dw_descripter'));
-
- $this->assertEquals($response[1]['content_type'], 'application/json');
- $this->assertEquals($response[1]['http_code'], 401);
+ );
}
/**
diff --git a/tests/integration/lib/Controllers/ReportBuilderTest.php b/tests/integration/lib/Controllers/ReportBuilderTest.php
index 4218ce8922..9034655e7d 100644
--- a/tests/integration/lib/Controllers/ReportBuilderTest.php
+++ b/tests/integration/lib/Controllers/ReportBuilderTest.php
@@ -3,7 +3,6 @@
namespace IntegrationTests\Controllers;
use CCR\Json;
-use TestHarness\TestFiles;
use TestHarness\XdmodTestHelper;
use IntegrationTests\BaseTest;
@@ -45,11 +44,6 @@ class ReportBuilderTest extends BaseTest
*/
protected $helper;
- /**
- * @var TestFiles
- */
- protected $testFiles;
-
/**
* @var bool
*/
@@ -60,19 +54,6 @@ class ReportBuilderTest extends BaseTest
'http_code' => 200
);
-
- /**
- * @return TestFiles
- * @throws \Exception
- */
- protected function getTestFiles()
- {
- if (!isset($this->testFiles)) {
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
- }
- return $this->testFiles;
- }
-
protected function setUp()
{
$this->verbose = getenv('TEST_VERBOSE');
@@ -192,7 +173,7 @@ public function testEnumAvailableCharts(array $options)
$this->assertEquals($expected['content_type'], $response[1]['content_type']);
$this->assertEquals($expected['http_code'], $response[1]['http_code']);
- $expectedFileName = $this->getTestFiles()->getFile('report_builder', $output['name'], 'output', $output['extension']);
+ $expectedFileName = parent::getTestFiles()->getFile('report_builder', $output['name'], 'output', $output['extension']);
if ($output['extension'] === '.json') {
$expected = JSON::loadFile($expectedFileName);
} else {
@@ -215,7 +196,7 @@ public function testEnumAvailableCharts(array $options)
public function provideEnumAvailableCharts()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('report_builder', 'enum_available_charts', 'input')
+ parent::getTestFiles()->getFile('report_builder', 'enum_available_charts', 'input')
);
}
@@ -253,7 +234,7 @@ public function testEnumReports(array $options)
$this->assertEquals($expected['content_type'], $response[1]['content_type']);
$this->assertEquals($expected['http_code'], $response[1]['http_code']);
- $expectedFileName = $this->getTestFiles()->getFile('report_builder', $output['name'], 'output', $output['extension']);
+ $expectedFileName = parent::getTestFiles()->getFile('report_builder', $output['name'], 'output', $output['extension']);
if ($output['extension'] === '.json') {
$expected = JSON::loadFile($expectedFileName);
} else {
@@ -277,7 +258,7 @@ public function testEnumReports(array $options)
public function provideEnumReports()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('report_builder', 'enum_reports', 'input')
+ parent::getTestFiles()->getFile('report_builder', 'enum_reports', 'input')
);
}
@@ -409,7 +390,7 @@ public function testCreateReport(array $options)
public function provideCreateReport()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('report_builder', 'create_report', 'input')
+ parent::getTestFiles()->getFile('report_builder', 'create_report', 'input')
);
}
@@ -428,7 +409,7 @@ public function testCreateChart(array $options)
$success = $options['success'];
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('report_builder', $expectedFile)
+ parent::getTestFiles()->getFile('report_builder', $expectedFile)
);
if ($user !== 'pub') {
@@ -455,7 +436,7 @@ public function testCreateChart(array $options)
public function provideCreateChart()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('report_builder', 'create_chart', 'input')
+ parent::getTestFiles()->getFile('report_builder', 'create_chart', 'input')
);
}
@@ -489,7 +470,7 @@ public function testEnumTemplates(array $options)
$this->assertEquals($expectedContentType, $curlinfo['content_type']);
$expected = Json::loadFile(
- $this->getTestFiles()->getFile('report_builder', $expectedFile)
+ parent::getTestFiles()->getFile('report_builder', $expectedFile)
);
$this->assertEquals($expected, $content);
@@ -506,7 +487,7 @@ public function testEnumTemplates(array $options)
public function provideEnumTemplates()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('report_builder', 'enum_templates', 'input')
+ parent::getTestFiles()->getFile('report_builder', 'enum_templates', 'input')
);
}
diff --git a/tests/integration/lib/Controllers/UsageExplorerTest.php b/tests/integration/lib/Controllers/UsageExplorerTest.php
index af96adb0fe..6d08a675d9 100644
--- a/tests/integration/lib/Controllers/UsageExplorerTest.php
+++ b/tests/integration/lib/Controllers/UsageExplorerTest.php
@@ -836,7 +836,7 @@ public function provideFilterIdLookup()
//TODO: Needs further integration for storage realm
$realmData = array();
- if (in_array("cloud", self::getRealms())) {
+ if (in_array("cloud", parent::getRealms())) {
array_push(
$realmData,
// Cloud, single value filter tests
@@ -884,7 +884,7 @@ public function provideFilterIdLookup()
);
};
- if (in_array("jobs", self::getRealms())) {
+ if (in_array("jobs", parent::getRealms())) {
array_push(
$realmData,
// Jobs, single value filter tests
diff --git a/tests/integration/lib/Controllers/UserAdminTest.php b/tests/integration/lib/Controllers/UserAdminTest.php
index 34a213ffa8..c9c1b770c9 100644
--- a/tests/integration/lib/Controllers/UserAdminTest.php
+++ b/tests/integration/lib/Controllers/UserAdminTest.php
@@ -3,27 +3,10 @@
namespace IntegrationTests\Controllers;
use CCR\Json;
-use JsonSchema\Validator;
use Models\Services\Realms;
-use TestHarness\TestFiles;
class UserAdminTest extends BaseUserAdminTest
{
-
- protected $testFiles;
-
- /**
- * @return TestFiles
- * @throws \Exception
- */
- public function getTestFiles()
- {
- if (!isset($this->testFiles)) {
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
- }
- return $this->testFiles;
- }
-
/**
* Tests all of the situations in which user creation can fail ( i.e. throw an exception ).
*
@@ -189,7 +172,7 @@ public function testCreateUsersSuccess(array $user)
public function provideCreateUsersSuccess()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'create_users', 'input')
+ parent::getTestFiles()->getFile('user_admin', 'create_users', 'input')
);
}
@@ -219,7 +202,7 @@ public function testThatExistingUsersCanBeRetrieved(array $user)
public function provideThatExistingUsersCanBeRetrieved()
{
return Json::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'existing_users', 'input')
+ parent::getTestFiles()->getFile('user_admin', 'existing_users', 'input')
);
}
@@ -283,13 +266,13 @@ function ($filter) use ($personId) {
public function provideTestUsersQuickFilters()
{
# Handle special case where there's no PI data (Cloud Realm enabled only)
- if (self::getRealms() == array("cloud")) {
+ if (parent::getRealms() == array("cloud")) {
return Json::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'user_quick_filters-update_enumAllAvailableRoles', 'output/cloud')
+ parent::getTestFiles()->getFile('user_admin', 'user_quick_filters-update_enumAllAvailableRoles', 'output/cloud')
);
} else {
return Json::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'user_quick_filters-update_enumAllAvailableRoles', 'output')
+ parent::getTestFiles()->getFile('user_admin', 'user_quick_filters-update_enumAllAvailableRoles', 'output')
);
}
}
@@ -326,18 +309,12 @@ public function testGetMenus(array $user)
$actual = $response[0];
- # Check spec file
- $schemaObject = JSON::loadFile(
- $this->getTestFiles()->getFile('schema', 'get-menus.spec', ''),
- false
- );
-
- $this->validateJson($actual, $schemaObject);
+ $this->validateJson($actual, 'schema', 'get-menus.spec', '');
# Check expected file
$expected = array();
foreach(self::$XDMOD_REALMS as $realm) {
- $expectedOutputFile = $this->getTestFiles()->getFile('user_admin', $output, "output/$realm");
+ $expectedOutputFile = parent::getTestFiles()->getFile('user_admin', $output, "output/$realm");
if(!is_file($expectedOutputFile)) {
$newFile = array();
@@ -384,7 +361,7 @@ public function testGetMenus(array $user)
public function provideGetMenus()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'get_menus-1', 'input')
+ parent::getTestFiles()->getFile('user_admin', 'get_menus-1', 'input')
);
}
@@ -428,9 +405,9 @@ public function testGetTabs(array $user)
$this->assertArrayHasKey('tabs', $actual['data'][0], '"data" has one element with "tabs"');
if (!in_array("jobs", self::$XDMOD_REALMS)) {
- $expectedFileName = $this->getTestFiles()->getFile('user_admin', $user['output'], 'output');
+ $expectedFileName = parent::getTestFiles()->getFile('user_admin', $user['output'], 'output');
} else {
- $expectedFileName = $this->getTestFiles()->getFile('user_admin', $user['output'], 'output/jobs');
+ $expectedFileName = parent::getTestFiles()->getFile('user_admin', $user['output'], 'output/jobs');
}
if (!is_file($expectedFileName)) {
@@ -452,7 +429,7 @@ public function testGetTabs(array $user)
public function provideGetTabs()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'get_tabs', 'input')
+ parent::getTestFiles()->getFile('user_admin', 'get_tabs', 'input')
);
}
@@ -478,11 +455,7 @@ public function testGetDwDescripters($username)
$response = $this->helper->post('controllers/metric_explorer.php', null, $data);
$actual = $response[0];
- $schemaObject = JSON::loadFile(
- $this->getTestFiles()->getFile('schema', 'dw_descripter.spec', ''),
- false
- );
- $this->validateJson($actual, $schemaObject);
+ $this->validateJson($actual, 'schema', 'dw_descripter.spec', '');
$this->validateResponse($response);
if (!$isPublicUser) {
$this->helper->logout();
@@ -545,7 +518,7 @@ public function testGetUserVisits(array $options)
$actual = json_decode($response[0], true);
$expected = JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', $expectedOutput, 'output')
+ parent::getTestFiles()->getFile('user_admin', $expectedOutput, 'output')
);
@@ -658,7 +631,7 @@ public function testGetUserVisitsExport(array $options)
}
$fileType = $expectedSuccess ? '.csv' : '.json';
- $expectedFileName = $this->getTestFiles()->getFile('user_admin', $expectedOutput, 'output', $fileType);
+ $expectedFileName = parent::getTestFiles()->getFile('user_admin', $expectedOutput, 'output', $fileType);
$rows = 0;
$length = 1;
@@ -764,7 +737,7 @@ function ($key, $expectedRow) use ($actualRow, $ignoredColumns) {
public function provideGetUserVisits()
{
$data = JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'get_user_visits', 'input')
+ parent::getTestFiles()->getFile('user_admin', 'get_user_visits', 'input')
);
$helper = new \TestHarness\XdmodTestHelper();
@@ -825,7 +798,7 @@ public function testGetUserVisitsIncrements(array $options)
public function provideGetUserVisitsIncrements()
{
return JSON::loadFile(
- $this->getTestFiles()->getFile('user_admin', 'get_user_visits_increment', 'input')
+ parent::getTestFiles()->getFile('user_admin', 'get_user_visits_increment', 'input')
);
}
diff --git a/tests/integration/lib/Controllers/UserControllerProviderTest.php b/tests/integration/lib/Controllers/UserControllerProviderTest.php
index 136a5b41d8..3d33155637 100644
--- a/tests/integration/lib/Controllers/UserControllerProviderTest.php
+++ b/tests/integration/lib/Controllers/UserControllerProviderTest.php
@@ -6,26 +6,11 @@
use Exception;
use Models\Services\Tokens;
use stdClass;
-use TestHarness\TokenHelper;
use TestHarness\Utilities;
use TestHarness\XdmodTestHelper;
class UserControllerProviderTest extends BaseUserAdminTest
{
- /**
- * @var TokenHelper
- */
- private $tokenHelper;
-
- /**
- * @throws Exception
- */
- protected function setUp()
- {
- parent::setUp();
- $this->tokenHelper = new TokenHelper(new XdmodTestHelper());
- }
-
/**
* Tests the UserControllerProvider route: GET /users/current
*
@@ -65,234 +50,27 @@ public function testGetCurrentUser(array $options)
}
/**
- * @dataProvider provideTestAPITokens
- * @param array $options
- * @return void
- * @throws Exception
+ * @dataProvider provideBaseRoles
*/
- public function testAPITokensCRD(array $options)
+ public function testAPITokensCRD($role)
{
- $hydratedOptions = $this->hydrateOptions($options, 'create_api_tokens');
-
- $user = $hydratedOptions->user;
- $expected = $hydratedOptions->expected;
-
- if ('pub' !== $user) {
- $this->tokenHelper->authenticate($user);
- }
-
- // Attempt to get the current API token, this should fail.
- $this->tokenHelper->getAPIToken(
- $expected->api_get->http_code,
- $expected->api_get->content_type,
- $expected->api_get->schemas->failure
- );
-
- // Attempt to create an API token.
- $this->tokenHelper->createAPIToken(
- $expected->api_create->http_code,
- $expected->api_create->content_type,
- $expected->api_create->schemas->success
- );
-
- // Now test that we can't create a token when we already have a valid token.
- $this->tokenHelper->createAPIToken(
- $expected->api_create->http_code,
- $expected->api_create->content_type,
- $expected->api_create->schemas->failure
- );
-
- // Now test if we can get the newly created token, this should succeed.
- $this->tokenHelper->getAPIToken(
- $expected->api_get->http_code,
- $expected->api_get->content_type,
- $expected->api_get->schemas->success
- );
-
- // Now we can revoke the token we just created.
- $this->tokenHelper->revokeAPIToken(
- $expected->api_revoke->http_code,
- $expected->api_revoke->content_type,
- $expected->api_revoke->schemas->success
- );
-
- // We cannot revoke a token if we don't have one.
- $this->tokenHelper->revokeAPIToken(
- $expected->api_revoke->http_code,
- $expected->api_revoke->content_type,
- $expected->api_revoke->schemas->failure
- );
-
- // We still can't get a token if we don't have one.
- $this->tokenHelper->getAPIToken(
- $expected->api_get->http_code,
- $expected->api_get->content_type,
- $expected->api_get->schemas->failure
- );
-
- if ('pub' !== $user) {
- $this->tokenHelper->logout();
- }
- }
-
- /**
- * This tests that API Token authentication is working for the controller operation:
- * `html/controllers/metric_explorer.php?operation=get_dw_descripter`.
- *
- * @dataProvider provideTestControllerTokenAuthentication
- * @param array $options
- * @return void
- * @throws Exception
- */
- public function testControllerTokenAuthentication(array $options)
- {
- $this->endpointTokenAuthenticationTest($options, 'test_endpoint_token_auth');
- }
-
- /**
- * This tests that API Token authentication is working for the Warehouse Export REST Controllers getRealms endpoint:
- * `rest/warehouse/export/realms`.
- *
- * @dataProvider provideTestRestTokenAuthentication
- *
- * @param array $options
- * @return void
- * @throws Exception
- */
- public function testRestTokenAuthentication(array $options)
- {
- $this->endpointTokenAuthenticationTest($options, 'test_rest_token_auth');
- }
-
- /**
- * A helper function that will test the end point provided with $options for it's ability to support API Token
- * authentication. Currently API Token authentication is supported for both controllers/* and Rest endpoints. This
- * is accomplished by a user logging into XDMoD normally and generating an API Token. Then, when making a request to
- * and endpoint that supports API Token authentication they include this token as a request header
- * 'Bearer: `.
- *
- * The steps of this test are as follows:
- * - Attempt a request to the endpoint located at $options->test->url before authorization. This is expected to fail.
- * - If we're not testing the public user then log the test user.
- * - Create a new API token for the currently logged in user.
- * - Add the newly created token to $this->helpers->headers so that it will be included when making requests.
- * - Attempt a request to the test endpoint again this time w/ the token in the headers.
- * - Verify that the results of the API Token authenticated request are as expected.
- * - Begin cleanup by clearing the API Token from the helpers headers.
- * - Revoke the previously created API Token.
- * - and finally, if the test user is not the public user then log them out.
- *
- * @param array $options
- * @param string $testId
- *
- * @return void
- *
- * @throws Exception
- */
- private function endpointTokenAuthenticationTest(array $options, $testId)
- {
- $hydratedOptions = $this->hydrateOptions($options, $testId);
-
- $user = $hydratedOptions->user;
- $expected = $hydratedOptions->expected;
- $test = $hydratedOptions->test;
-
- // Attempt to make a request to the controller endpoint unauthenticated in any way.
- // This should fail. ( We use the tokenHelper so that we can specify a schema that the response should be
- // validated against. )
- $this->tokenHelper->makeRequest(
- $test->description,
- $test->url,
- $test->verb,
- $test->parameters,
- $test->data,
- $expected->test->failure->http_code,
- $expected->test->failure->content_type,
- $expected->test->failure->schema
- );
-
- // Now go ahead and authenticate the test user so we can create / use their API Token.
- if ('pub' !== $user) {
- $this->tokenHelper->authenticate($user);
- }
-
- // Attempt to create an API token.
- $tokenResponse = $this->tokenHelper->createAPIToken(
- $expected->api_create->http_code,
- $expected->api_create->content_type,
- $expected->api_create->schemas->success
- );
-
- if ('pub' !== $user) {
- $token = $tokenResponse->data->token;
- // We add the token to the request headers so that we can use the token authentication.
- $this->helper->addheader('Authorization', sprintf('%s %s', Tokens::HEADER_KEY, $token));
-
- // While we prefer the 'Authorization' header approach to token authentication. We also support having it
- // provided as a query parameter. Also, retrieving the 'Authorization' header on el7 isn't working, so
- // this is the only way to test authentication.
- if (!property_exists($test, 'parameters') || !isset($test->parameters)) {
- $test->parameters = array();
- } elseif (is_object($test->parameters)) {
- $test->parameters = (array)$test->parameters;
- }
-
- $test->parameters[Tokens::HEADER_KEY] = $token;
-
- $verb = $test->verb;
- $response = $this->helper->$verb($test->url, (array)$test->parameters, (array)$test->data);
-
- $this->assertEquals($expected->test->success->http_code, $response[1]['http_code']);
- $this->assertEquals($expected->test->success->content_type, $response[1]['content_type']);
-
- $expectedOutput = file_get_contents(
- $this->getTestFiles()->getFile(
- $expected->test->success->file_group,
- $expected->test->success->file_name,
- 'output',
- $expected->test->success->file_ext
- )
- );
-
- // need to make sure that we format the expected output to match the actual output.
- if ($expected->test->success->file_ext === '.json') {
- $expectedOutput = json_decode($expectedOutput, true);
+ if ('pub' === $role) {
+ foreach (array('get', 'create', 'revoke') as $action) {
+ $this->makeTokenRequest($action, 401, 'authentication_error');
}
-
- $this->assertEquals($expectedOutput, $response[0]);
-
- // clean up the helper's headers.
- $this->helper->addheader('Authorization', null);
- }
-
- // Make sure to revoke the token so that we leave the user in the same state as we found it.
- $this->tokenHelper->revokeAPIToken(
- $expected->api_revoke->http_code,
- $expected->api_revoke->content_type,
- $expected->api_revoke->schemas->success
- );
-
- // And finally make sure that we log out.
- if ('pub' !== $user) {
- $this->tokenHelper->logout();
+ } else {
+ $this->helper->authenticate($role);
+ $this->helper->delete('rest/users/current/api/token');
+ $this->makeTokenRequest('get', 404, 'not_found');
+ $this->makeTokenRequest('create', 200, 'create_success', 'schema');
+ $this->makeTokenRequest('create', 409, 'create_failure');
+ $this->makeTokenRequest('get', 200, 'get_success', 'schema');
+ $this->makeTokenRequest('revoke', 200, 'revoke_success');
+ $this->makeTokenRequest('revoke', 404, 'not_found');
+ $this->helper->logout();
}
}
- /**
- * @param array $options
- * @param string $testId the id of the test calling this function, will be used to retrieve the default test options
- * @return stdClass containing the provided options merged w/ token_auth_defaults and the defaults for $testId.
- * @throws Exception
- */
- protected function hydrateOptions(array $options, $testId)
- {
- $tokenAuthDefaults = Json::loadFile($this->getTestFiles()->getFile('user_controller', 'token_auth_defaults', 'input'), false);
- $testDefaults = Json::loadFile($this->getTestFiles()->getFile('user_controller', sprintf('%s_defaults', $testId), 'input'), false);
-
- $authedOptions = Utilities::applyDefaults(json_decode(json_encode($options)), $tokenAuthDefaults);
- return Utilities::applyDefaults($authedOptions, $testDefaults);
- }
-
/**
* @return array|object
* @throws Exception
@@ -304,24 +82,30 @@ public function provideGetCurrentUser()
);
}
- public function provideTestAPITokens()
- {
- return JSON::loadFile(
- $this->getTestFiles()->getFile('user_controller', 'create_api_tokens', 'input')
- );
- }
-
- public function provideTestControllerTokenAuthentication()
- {
- return JSON::loadFile(
- $this->getTestFiles()->getFile('user_controller', 'test_endpoint_token_auth', 'input')
- );
- }
-
- public function provideTestRestTokenAuthentication()
- {
- return JSON::loadFile(
- $this->getTestFiles()->getFile('user_controller', 'test_endpoint_token_auth', 'input')
+ private function makeTokenRequest(
+ $action,
+ $httpCode,
+ $outputFileName,
+ $validationType = 'exact'
+ ) {
+ if ('get' === $action) {
+ $verb = 'get';
+ } elseif ('create' === $action) {
+ $verb = 'post';
+ } else {
+ $verb = 'delete';
+ }
+ return $this->makeRequest(
+ $this->helper,
+ 'rest/users/current/api/token',
+ $verb,
+ null,
+ null,
+ $httpCode,
+ 'application/json',
+ 'integration/rest/user/api_token',
+ $outputFileName . ($validationType === 'schema' ? '.spec' : ''),
+ $validationType
);
}
}
diff --git a/tests/integration/lib/Controllers/UserInterfaceTest.php b/tests/integration/lib/Controllers/UserInterfaceTest.php
index 0c0867e3d8..0a5c1cb23c 100644
--- a/tests/integration/lib/Controllers/UserInterfaceTest.php
+++ b/tests/integration/lib/Controllers/UserInterfaceTest.php
@@ -54,12 +54,7 @@ public function testGetMenus(array $options)
$actual = $response[0];
- # Check spec file
- $schemaObject = JSON::loadFile(
- $this->getTestFiles()->getFile('schema', 'get-menus.spec', ''),
- false
- );
- $this->validateJson($actual, $schemaObject);
+ $this->validateJson($actual, 'schema', 'get-menus.spec', '');
# Check expected file
$expected = array();
@@ -99,7 +94,7 @@ public function testGetMenus(array $options)
$this->assertEquals($expected, $actual);
- } // public function testGetMenus(array $options)
+ }
/**
* Provides test data to `testGetMenus`.
@@ -112,5 +107,5 @@ public function provideTestGetMenus()
return JSON::loadFile(
$this->getTestFiles()->getFile('user_interface', 'get_menus', 'input')
);
- } // public function provideTestGetMenus()
+ }
}
diff --git a/tests/integration/lib/Database/BaseDatabaseTest.php b/tests/integration/lib/Database/BaseDatabaseTest.php
index 0e5b062aa2..34f8857f05 100644
--- a/tests/integration/lib/Database/BaseDatabaseTest.php
+++ b/tests/integration/lib/Database/BaseDatabaseTest.php
@@ -7,21 +7,17 @@
use Configuration\XdmodConfiguration;
use Exception;
use IntegrationTests\BaseTest;
-use TestHarness\TestFiles;
abstract class BaseDatabaseTest extends BaseTest
{
protected $db;
- protected $testFiles;
-
/**
* @throws Exception
*/
public function setUp()
{
$this->db = DB::factory('datawarehouse');
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
}
/**
@@ -55,17 +51,16 @@ protected function validateDatabaseValues(
) {
$actual = $this->db->query($actualSQLQuery);
- # Check spec file
- $schemaObject = Json::loadFile(
- $this->testFiles->getFile($schemaTestGroup, $actualSchemaFileName, ''),
- false
+ $this->validateJson(
+ $actual,
+ $schemaTestGroup,
+ $actualSchemaFileName,
+ ''
);
- $this->validateJson($actual, $schemaObject);
-
# Check expected file
foreach(self::$XDMOD_REALMS as $realm) {
- $expectedOutputFile = $this->testFiles->getFile($expectedTestGroup, $expectedFileName, "output/" . strtolower($realm));
+ $expectedOutputFile = parent::getTestFiles()->getFile($expectedTestGroup, $expectedFileName, "output/" . strtolower($realm));
# Create missing files/directories
if(!is_file($expectedOutputFile)) {
diff --git a/tests/integration/lib/Database/ResourceNamesTest.php b/tests/integration/lib/Database/ResourceNamesTest.php
index 0d900108c9..5064e7aa7d 100644
--- a/tests/integration/lib/Database/ResourceNamesTest.php
+++ b/tests/integration/lib/Database/ResourceNamesTest.php
@@ -6,7 +6,6 @@
use CCR\DB;
use CCR\Json;
use PHPUnit_Framework_TestCase;
-use TestHarness\TestFiles;
use Configuration\XdmodConfiguration;
/**
@@ -16,29 +15,25 @@ class ResourceNamesTest extends BaseTest
{
private $db;
- private $testFiles;
-
public function setUp()
{
$this->db = DB::factory('datawarehouse');
- $this->testFiles = new TestFiles(__DIR__ . '/../../../');
}
public function testResourcesNamesValues()
{
$actual = $this->db->query('SELECT code, name FROM modw.resourcefact ORDER BY code');
- # Check spec file
- $schemaObject = Json::loadFile(
- $this->testFiles->getFile('schema/integration', 'resource_names.spec', ''),
- false
+ $this->validateJson(
+ $actual,
+ 'schema/integration',
+ 'resource_names.spec',
+ ''
);
- $this->validateJson($actual, $schemaObject);
-
# Check expected file
foreach(self::$XDMOD_REALMS as $realm) {
- $expectedOutputFile = $this->testFiles->getFile('integration/database', 'resource_names', "output/" . strtolower($realm));
+ $expectedOutputFile = parent::getTestFiles()->getFile('integration/database', 'resource_names', "output/" . strtolower($realm));
# Create missing files/directories
if(!is_file($expectedOutputFile)) {
diff --git a/tests/integration/lib/Database/SharedJobsTest.php b/tests/integration/lib/Database/SharedJobsTest.php
index a65a05e4af..d1ab3fdc09 100644
--- a/tests/integration/lib/Database/SharedJobsTest.php
+++ b/tests/integration/lib/Database/SharedJobsTest.php
@@ -5,8 +5,6 @@
use CCR\DB;
use CCR\Json;
use PHPUnit_Framework_TestCase;
-use TestHarness\TestFiles;
-use JsonSchema\Validator;
use IntegrationTests\BaseTest;
use Configuration\XdmodConfiguration;
diff --git a/tests/integration/lib/Rest/JobViewerTest.php b/tests/integration/lib/Rest/JobViewerTest.php
index 068cca7093..2b230dcafe 100644
--- a/tests/integration/lib/Rest/JobViewerTest.php
+++ b/tests/integration/lib/Rest/JobViewerTest.php
@@ -268,7 +268,7 @@ public function testInvalidJobSearchJson() {
public function missingParamsProvider() {
//TODO: Needs further integration for other realms.
- if (!in_array("jobs", self::getRealms())) {
+ if (!in_array("jobs", parent::getRealms())) {
$this->markTestSkipped('Needs realm integration.');
}
diff --git a/tests/integration/lib/Rest/WarehouseControllerTest.php b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
similarity index 98%
rename from tests/integration/lib/Rest/WarehouseControllerTest.php
rename to tests/integration/lib/Rest/WarehouseControllerProviderTest.php
index 8aa48574f7..62549f75d6 100644
--- a/tests/integration/lib/Rest/WarehouseControllerTest.php
+++ b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
@@ -4,7 +4,7 @@
use IntegrationTests\BaseTest;
-class WarehouseControllerTest extends BaseTest
+class WarehouseControllerProviderTest extends BaseTest
{
protected static $helpers = array();
diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerProviderTest.php
similarity index 82%
rename from tests/integration/lib/Rest/WarehouseExportControllerTest.php
rename to tests/integration/lib/Rest/WarehouseExportControllerProviderTest.php
index 6769c275a5..8989f7879a 100644
--- a/tests/integration/lib/Rest/WarehouseExportControllerTest.php
+++ b/tests/integration/lib/Rest/WarehouseExportControllerProviderTest.php
@@ -5,10 +5,11 @@
use CCR\DB;
use DataWarehouse\Export\FileManager;
use DataWarehouse\Export\QueryHandler;
+use IntegrationTests\BaseTest;
use JsonSchema\Constraints\Constraint;
use JsonSchema\Validator;
use PHPUnit_Framework_TestCase;
-use TestHarness\TestFiles;
+use TestHarness\TokenHelper;
use TestHarness\XdmodTestHelper;
use XDUser;
@@ -17,12 +18,12 @@
*
* @coversDefaultClass \Rest\Controllers\WarehouseExportControllerProvider
*/
-class WarehouseExportControllerTest extends PHPUnit_Framework_TestCase
+class WarehouseExportControllerProviderTest extends BaseTest
{
/**
* Test files base path.
*/
- const TEST_GROUP = 'integration/rest/warehouse-export';
+ const TEST_GROUP = 'integration/rest/warehouse/export';
/**
* User roles and usernames.
@@ -79,23 +80,6 @@ class WarehouseExportControllerTest extends PHPUnit_Framework_TestCase
*/
private static $schemaCache = [];
- /**
- * @var TestFiles
- */
- private static $testFiles;
-
- /**
- * @return TestFiles
- */
- private static function getTestFiles()
- {
- if (!isset(self::$testFiles)) {
- self::$testFiles = new TestFiles(__DIR__ . '/../../../');
- }
-
- return self::$testFiles;
- }
-
/**
* Instantiate fixtures and authenticate helpers.
*/
@@ -150,10 +134,10 @@ public static function tearDownAfterClass()
private static function getSchema($schema)
{
if (!array_key_exists($schema, self::$schemaCache)) {
- static::$schemaCache[$schema] = self::getTestFiles()->loadJsonFile(
+ static::$schemaCache[$schema] = parent::getTestFiles()->loadJsonFile(
'schema',
$schema . '.schema',
- 'warehouse-export',
+ 'warehouse/export',
false
);
}
@@ -202,35 +186,35 @@ function ($error) {
}
/**
- * Test getting the list of exportable realms.
- *
- * @param string $role Role to use during test.
- * @param int $httpCode Expected HTTP response code.
- * @param string $schema Name of JSON schema file that will be used
- * to validate returned data.
- * @param array $realms The name of the realms that are expected to
- * be in the returned data.
* @covers ::getRealms
- * @dataProvider getRealmsProvider
+ * @dataProvider provideBaseRoles
*/
- public function testGetRealms($role, $httpCode, $schema, array $realms)
+ public function testGetRealms($role)
{
- list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/realms');
- $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
- $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code');
- $this->validateAgainstSchema($content, $schema);
-
- // Only check data for successful requests.
- if ($httpCode == 200) {
- // Testing each realm individually to avoid putting field
- // definitions in test artifacts.
- $this->assertTrue(is_array($content), 'Content is an array');
- $this->assertArrayHasKey('data', $content, 'Content has a "data" key');
- $this->assertTrue(is_array($content['data']), 'Data is an array');
- $this->assertCount(count($realms), $content['data'], 'Data contains correct number of realms');
- foreach ($content['data'] as $i => $realm) {
- $this->assertArraySubset($realms[$i], $realm, sprintf('Realm %d contains the expected subset', $i + 1));
+ $tokenHelper = new TokenHelper(
+ $this,
+ self::$helpers[$role],
+ $role,
+ 'rest/warehouse/export/realms',
+ 'get',
+ null,
+ null,
+ 'rest',
+ 'token_optional'
+ );
+ $tokenHelper->runEndpointTests(
+ function ($token) use ($tokenHelper) {
+ $tokenHelper->runEndpointTest(
+ $token,
+ 'get_realms.spec',
+ 200,
+ 'integration/rest/warehouse/export',
+ 'schema'
+ );
}
+ );
+ if ('pub' !== $role) {
+ self::$helpers[$role]->authenticate($role);
}
}
@@ -409,33 +393,28 @@ public function testDeleteRequests($role, $httpCode, $schema)
$this->assertEquals([], $afterContent['data'], 'Data after deletion is empty.');
}
- public function getRealmsProvider()
- {
- return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-realms', 'input');
- }
-
public function createRequestProvider()
{
- return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'create-request', 'input');
+ return parent::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'create-request', 'input');
}
public function getRequestsProvider()
{
- return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-requests', 'input');
+ return parent::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-requests', 'input');
}
public function getRequestProvider()
{
- return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-request', 'input');
+ return parent::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-request', 'input');
}
public function deleteRequestProvider()
{
- return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-request', 'input');
+ return parent::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-request', 'input');
}
public function deleteRequestsProvider()
{
- return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-requests', 'input');
+ return parent::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-requests', 'input');
}
}
diff --git a/tests/integration/lib/TestHarness/TokenHelper.php b/tests/integration/lib/TestHarness/TokenHelper.php
index 4aaeef358c..463d4e2eb0 100644
--- a/tests/integration/lib/TestHarness/TokenHelper.php
+++ b/tests/integration/lib/TestHarness/TokenHelper.php
@@ -3,261 +3,213 @@
namespace TestHarness;
use CCR\DB;
-use CCR\Json;
-use Exception;
+use Models\Services\Tokens;
/**
*
*/
class TokenHelper
{
-
- /**
- * @var XdmodTestHelper
- */
- private $helper;
-
- /**
- * @var TestFiles
- */
- private $testFiles;
-
-
- /**
- *
- * @param XdmodTestHelper $helper
- * @param TestFiles $testFiles
- * @throws Exception
- */
- public function __construct($helper = null, $testFiles = null)
- {
- // if we aren't passed an XdmodTestHelper instance then create one ourselves.
- if (!isset($helper)) {
- $helper = new XdmodTestHelper();
- }
-
- $this->helper = $helper;
-
- if (!isset($testFiles)) {
- $testFiles = new TestFiles(__DIR__ . '/../../../');
+ private static $ENDPOINT = 'rest/users/current/api/token';
+ private static $TEST_GROUP = 'integration/rest/user/api_token';
+ private $testInstance;
+ private $testHelper;
+ private $role;
+ private $path;
+ private $verb;
+ private $params;
+ private $data;
+ private $expectedOutputs = array(
+ 'empty_token' => array(),
+ 'malformed_token' => array(),
+ 'invalid_token' => array(),
+ 'expired_token' => array()
+ );
+
+ public function __construct(
+ $testInstance,
+ $testHelper,
+ $role,
+ $path,
+ $verb,
+ $params,
+ $data,
+ $endpointType,
+ $authenticationType
+ ) {
+ $this->testInstance = $testInstance;
+ $this->testHelper = $testHelper;
+ $this->role = $role;
+ $this->path = $path;
+ $this->verb = $verb;
+ $this->params = $params;
+ $this->data = $data;
+ if ('token_optional' === $authenticationType) {
+ foreach (array_keys($this->expectedOutputs) as $type) {
+ if ('controller' === $endpointType) {
+ $fileName = 'session_expired';
+ } elseif ('rest' === $endpointType) {
+ $fileName = 'authentication_error';
+ }
+ $this->setExpectedErrorOutput($type, $fileName);
+ }
+ } elseif ('token_required' === $authenticationType) {
+ foreach (array(
+ 'empty_token',
+ 'malformed_token',
+ 'invalid_token',
+ 'expired_token'
+ ) as $type) {
+ $this->setExpectedErrorOutput(
+ $type,
+ $type,
+ array(
+ 'WWW-Authenticate' => Tokens::HEADER_KEY
+ )
+ );
+ }
}
- $this->testFiles = $testFiles;
- }
-
- /**
- * Attempt to retrieve the metadata about the currently logged in users API Token. If any of the $expected* arguments
- * are provided then the function will attempt to validate that they match what is returned by the endpoint.
- *
- * @param int $expectedHttpCode
- * @param string $expectedContentType
- * @param string $expectedSchemaFileName
- * @return mixed
- * @throws Exception
- */
- public function getAPIToken($expectedHttpCode = null, $expectedContentType = null, $expectedSchemaFileName = null)
- {
- return $this->makeRequest(
- 'Get API Token',
- 'rest/users/current/api/token',
- 'get',
- null,
- null,
- $expectedHttpCode,
- $expectedContentType,
- $expectedSchemaFileName
- );
}
- /**
- * Attempt to create a new API Token for the currently logged in user.
- *
- * If any of the $expected* arguments are included than they will be used to validate the information returned from
- * the endpoint.
- *
- * @param $expectedHttpCode
- * @param $expectedContentType
- * @param $expectedSchemaName
- * @return object containing the api token value
- * @throws Exception
- */
- public function createAPIToken($expectedHttpCode = null, $expectedContentType = null, $expectedSchemaName = null)
- {
- return $this->makeRequest(
- 'Create API Token',
- 'rest/users/current/api/token',
- 'post',
- null,
- null,
- $expectedHttpCode,
- $expectedContentType,
- $expectedSchemaName
- );
- }
-
-
- /**
- * Attempt to revoke the API token for the currently logged in user.
- *
- * If any of the $expected* arguments are included than they will be used to validate the information returned from
- * the endpoint.
- *
- *
- * @param int $expectedHttpCode
- * @param string $expectedContentType
- * @param string $expectedSchemaFileName
- * @return mixed the response body
- * @throws Exception
- */
- public function revokeAPIToken($expectedHttpCode = null, $expectedContentType = null, $expectedSchemaFileName = null)
+ public function runEndpointTests($callback)
{
- return $this->makeRequest(
- 'Revoke API Token',
- 'rest/users/current/api/token',
- 'delete',
- null,
- null,
- $expectedHttpCode,
- $expectedContentType,
- $expectedSchemaFileName
- );
+ if ('pub' === $this->role) {
+ self::runStandardEndpointTest('', 'empty_token');
+ self::runStandardEndpointTest('asdf', 'malformed_token');
+ } else {
+ $this->testHelper->authenticate($this->role);
+ $this->testHelper->delete(self::$ENDPOINT);
+ $response = $this->testHelper->post(self::$ENDPOINT, null, null);
+ $token = $response[0]['data']['token'];
+ $userId = substr($token, 0, strpos($token, Tokens::DELIMITER));
+ $this->testHelper->logout();
+ self::runStandardEndpointTest(
+ $userId . Tokens::DELIMITER . 'asdf',
+ 'invalid_token'
+ );
+ $callback($token);
+ self::expireToken($userId);
+ self::runStandardEndpointTest($token, 'expired_token');
+ $this->testHelper->authenticate($this->role);
+ $this->testHelper->delete(self::$ENDPOINT);
+ $this->testHelper->logout();
+ self::runStandardEndpointTest($token, 'invalid_token');
+ }
}
- /**
- * @param string $endPointDescription
- * @param string $url
- * @param string $verb
- * @param array|null $params
- * @param array|null $data
- * @param int|null $expectedHttpCode
- * @param string|null $expectedContentType
- * @param string|null $expectedSchemaFileName
- * @return mixed
- * @throws Exception
- */
- public function makeRequest(
- $endPointDescription,
- $url,
- $verb,
- $params = null,
- $data = null,
- $expectedHttpCode = null,
- $expectedContentType = null,
- $expectedSchemaFileName = null
+ public function runEndpointTest(
+ $token,
+ $outputFileName = null,
+ $httpCode = null,
+ $outputTestGroup = null,
+ $validationType = 'exact',
+ $expectedHeaders = null
) {
- $response = null;
- switch ($verb) {
- case 'get':
- case 'put':
- $response = $this->helper->$verb($url, $params);
- break;
- case 'post':
- case 'delete':
- $response = $this->helper->$verb($url, $params, $data);
- break;
+ if (null === $outputTestGroup) {
+ $outputTestGroup = self::$TEST_GROUP;
}
- $actualHttpCode = isset($response) ? $response[1]['http_code'] : null;
- $actualContentType = isset($response) ? $response[1]['content_type'] : null;
- $actualResponseBody = isset($response) ? $response[0] : array();
-
- if (isset($expectedHttpCode) ) {
- // Note $expectedHttpCode was changed to support being an array due to el7 returning 400 where el8 returns
- // 401.
- if (is_numeric($expectedHttpCode) && $expectedHttpCode !== $actualHttpCode ||
- is_array($expectedHttpCode) && !in_array($actualHttpCode, $expectedHttpCode)
- ) {
- throw new Exception(
- sprintf(
- 'HTTP Code does not match. Expected: %s Received: %s',
- json_encode($expectedHttpCode),
- $actualHttpCode
- )
- );
- }
+ $defaultOutput = $this->expectedOutputs['empty_token'];
+ if (null === $outputFileName) {
+ $outputFileName = $defaultOutput['file_name'];
}
- if (isset($expectedContentType) && $expectedContentType !== $actualContentType) {
- print_r($response);
- throw new Exception(
- sprintf(
- 'HTTP Content Type does not match. Expected: %s Received: %s',
- $expectedContentType,
- $actualContentType
- )
- );
+ if (null === $httpCode) {
+ $httpCode = $defaultOutput['http_code'];
}
-
- $actual = json_decode(json_encode($actualResponseBody));
-
- if (isset($expectedSchemaFileName)) {
- $validator = new \JsonSchema\Validator();
- $expectedSchema = Json::loadFile(
- $this->testFiles->getFile('schema/integration', $expectedSchemaFileName, ''),
- false
+ $responseBodies = array();
+ foreach (array('token_in_header', 'token_not_in_header') as $mode) {
+ if ('token_in_header' === $mode) {
+ $authHeader = $this->testHelper->getheader('Authorization');
+ $this->testHelper->addheader(
+ 'Authorization',
+ Tokens::HEADER_KEY . ' ' . $token
+ );
+ }
+ if (null === $this->params) {
+ $this->params = array();
+ }
+ $this->params[Tokens::HEADER_KEY] = $token;
+ $responseBodies[$mode] = $this->testInstance->makeRequest(
+ $this->testHelper,
+ $this->path,
+ $this->verb,
+ $this->params,
+ $this->data,
+ $httpCode,
+ 'application/json',
+ $outputTestGroup,
+ $outputFileName,
+ $validationType,
+ $expectedHeaders
);
-
- $validator->validate($actual, $expectedSchema);
-
- if (!$validator->isValid()) {
- throw new Exception(sprintf(
- "%s response is in an invalid format.\nExpected:%s\nReceived %s",
- $endPointDescription,
- json_encode($expectedSchema, JSON_PRETTY_PRINT),
- var_export($actual, true)
- ));
+ if ('token_in_header' === $mode) {
+ $this->testHelper->addheader('Authorization', $authHeader);
}
+ unset($this->params[Tokens::HEADER_KEY]);
}
-
- return $actual;
+ $this->testInstance->assertSame(
+ json_encode($responseBodies['token_in_header']),
+ json_encode($responseBodies['token_not_in_header']),
+ json_encode(
+ $responseBodies['token_in_header'],
+ JSON_PRETTY_PRINT
+ )
+ . "\n"
+ . json_encode(
+ $responseBodies['token_not_in_header'],
+ JSON_PRETTY_PRINT
+ )
+ );
+ return $responseBodies['token_in_header'];
}
/**
* A helper function that will allow us to test the expiration of a token.
*
- * Note: We need to directly access the database as we do not have an endpoint for expiring
- * a token.
+ * Note: We need to directly access the database as we do not have an
+ * endpoint for expiring a token.
*
* @param string $userId the userId whose token should be expired.
*
* @return bool true if the token was successfully expired.
*
- * @throws Exception if there is a problem parsing the the provided $rawToken.
- * @throws Exception if there is a problem connecting to or executing the update statement against the database.
+ * @throws Exception if there is a problem parsing the the provided
+ * $rawToken.
+ * @throws Exception if there is a problem connecting to or executing the
+ * update statement against the database.
*/
- public function expireToken($userId)
+ public static function expireToken($userId)
{
$db = DB::factory('database');
- $query = 'UPDATE moddb.user_tokens SET expires_on = NOW() WHERE user_id = :user_id';
+ $query = 'UPDATE moddb.user_tokens SET expires_on = SUBDATE(NOW(), 1)'
+ . ' WHERE user_id = :user_id';
$params = array(':user_id' => $userId);
return $db->execute($query, $params) === 1;
}
-
- /**
- * Calls `authenticate($user)` on this TokenHelper instances XdmodTestHelper.
- *
- * @param string $user
- * @return void
- * @throws Exception
- */
- public function authenticate($user)
- {
- $this->helper->authenticate($user);
- }
-
- /**
- * Calls `logout` on this TokenHelper instances XdmodTestHelper.
- * @return void
- */
- public function logout()
- {
- $this->helper->logout();
+ private function setExpectedErrorOutput(
+ $type,
+ $fileName = null,
+ $expectedHeaders = null
+ ) {
+ if (null === $fileName) {
+ $fileName = $type;
+ }
+ $this->expectedOutputs[$type] = array(
+ 'http_code' => 401,
+ 'file_name' => $fileName,
+ 'expected_headers' => $expectedHeaders
+ );
}
- /**
- * Retrieve the XdmodTestHelper used by the TokenHelper.
- * @return XdmodTestHelper
- */
- public function getHelper()
+ private function runStandardEndpointTest($token, $type)
{
- return $this->helper;
+ $this->runEndpointTest(
+ $token,
+ $this->expectedOutputs[$type]['file_name'],
+ $this->expectedOutputs[$type]['http_code'],
+ self::$TEST_GROUP,
+ 'exact',
+ $this->expectedOutputs[$type]['expected_headers']
+ );
}
}
diff --git a/tests/integration/lib/TestHarness/Utilities.php b/tests/integration/lib/TestHarness/Utilities.php
index 1e82159cec..817c9e66f1 100644
--- a/tests/integration/lib/TestHarness/Utilities.php
+++ b/tests/integration/lib/TestHarness/Utilities.php
@@ -33,64 +33,4 @@ public static function getRealmsToTest()
}
return explode(',', $realm_list);
}
-
- /**
- * A helper function that will recursively merge the contents of $defaults to $source.
- *
- * @param stdClass|array $source
- * @param stdClass|array $defaults
- * @return stdClass|array
- */
- public static function applyDefaults($source, $defaults)
- {
- if (is_object($source) && is_object($defaults)) {
- return self::applyDefaultsObjects($source, $defaults);
- } elseif (is_array($source) && is_array($defaults)) {
- return self::applyDefaultsArray($source, $defaults);
- }
- return $source;
- }
-
- private static function applyDefaultsObjects(stdClass $source, stdClass $defaults) {
- foreach($defaults as $property => $defaultValue) {
- if (!isset($source->$property)) {
- $source->$property = $defaultValue;
- }
-
- $sourceValue = &$source->$property;
- if (is_object($sourceValue) && is_object($defaultValue)) {
- $source->$property = self::applyDefaults($sourceValue, $defaultValue);
- } elseif (is_array($sourceValue) && is_array($defaultValue)) {
- foreach($defaultValue as $default) {
- if (!in_array($default, $sourceValue)) {
- $sourceValue[] = $default;
- }
- }
- }
- // note, we do not want to replace scalar values if they already exist in $source.
- }
-
- return $source;
- }
-
- private static function applyDefaultsArray(array $source, array $defaults)
- {
- foreach($defaults as $property => $defaultValue) {
- if (!isset($source[$property])) {
- $source[$property] = $defaultValue;
- }
-
- $sourceValue = $source[$property];
- if (is_object($sourceValue) && is_object($defaultValue)) {
- $source[$property] = self::applyDefaults($sourceValue, $defaultValue);
- } elseif (is_array($sourceValue) && is_array($defaultValue)) {
- foreach($defaultValue as $default) {
- if (!in_array($default, $sourceValue)) {
- $sourceValue[] = $default;
- }
- }
- }
- }
- return $source;
- }
}
diff --git a/tests/integration/lib/TestHarness/XdmodTestHelper.php b/tests/integration/lib/TestHarness/XdmodTestHelper.php
index faaad3183c..1754aacae9 100644
--- a/tests/integration/lib/TestHarness/XdmodTestHelper.php
+++ b/tests/integration/lib/TestHarness/XdmodTestHelper.php
@@ -90,7 +90,7 @@ private function setauthvariables($token, $cookie = null)
}
}
- private function getheaders()
+ public function getheaders()
{
$headers = array();
foreach ($this->headers as $name => $value) {
From ab9c0e7f4f9f6e156a3f7de024aadced3ec548d2 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Thu, 20 Apr 2023 13:14:53 -0400
Subject: [PATCH 02/21] Add endpoints for `raw-data` and `raw-data/limit`.
---
classes/DataWarehouse/Data/BatchDataset.php | 50 +-
.../WarehouseControllerProvider.php | 198 ++++++-
configuration/portal_settings.ini | 1 +
docs/xdmod-rest-schema.json | 493 ++++++++++++++++--
.../xdmod/integration/base/error104.json | 4 +
.../output/get_raw_data/end_before_start.json | 4 +
.../get_raw_data/end_date_malformed.json | 4 +
.../output/get_raw_data/invalid_fields.json | 4 +
.../get_raw_data/invalid_filter_key.json | 4 +
.../output/get_raw_data/invalid_realm.json | 4 +
.../output/get_raw_data/limit_success.json | 4 +
.../output/get_raw_data/negative_offset.json | 4 +
.../output/get_raw_data/no_end_date.json | 4 +
.../output/get_raw_data/no_realm.json | 4 +
.../output/get_raw_data/no_start_date.json | 4 +
.../get_raw_data/start_date_malformed.json | 4 +
.../output/get_raw_data/success.spec.json | 30 ++
.../output/get_raw_data/success_0.spec.json | 14 +
.../get_raw_data/success_16500.spec.json | 14 +
.../get_raw_data/success_all_fields.spec.json | 20 +
.../success_fields_and_filters.spec.json | 25 +
.../Rest/WarehouseControllerProviderTest.php | 236 ++++++++-
.../lib/TestHarness/TokenHelper.php | 5 +
23 files changed, 1067 insertions(+), 67 deletions(-)
create mode 100644 tests/artifacts/xdmod/integration/base/error104.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json
diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php
index dc266e01ab..9753c50834 100644
--- a/classes/DataWarehouse/Data/BatchDataset.php
+++ b/classes/DataWarehouse/Data/BatchDataset.php
@@ -70,12 +70,35 @@ class BatchDataset extends Loggable implements Iterator
private $hashCache = [];
/**
- * @param mixed $name Description.
- * @param \XDUser $user
- * @param LoggerInterface $logger
+ * Maximum number of rows to return.
+ *
+ * @var int
*/
- public function __construct(RawQuery $query, XDUser $user, LoggerInterface $logger = null)
- {
+ private $limit;
+
+ /**
+ * Starting row index.
+ *
+ * @var int
+ */
+ private $offset;
+
+ /**
+ * @param RawQuery $query
+ * @param XDUser $user
+ * @param LoggerInterface $logger
+ * @param array $fields
+ * @param int $limit
+ * @param int $offset
+ */
+ public function __construct(
+ RawQuery $query,
+ XDUser $user,
+ LoggerInterface $logger = null,
+ $fields = null,
+ $limit = null,
+ $offset = 0
+ ) {
parent::__construct($logger);
$this->query = $query;
@@ -92,6 +115,21 @@ public function __construct(RawQuery $query, XDUser $user, LoggerInterface $logg
$rawStatsConfig = RawStatisticsConfiguration::factory();
$this->fields = $rawStatsConfig->getBatchExportFieldDefinitions($query->getRealmName());
+ if ($fields !== null) {
+ foreach ($this->fields as $index => $field) {
+ if (!in_array($field['alias'], $fields)) {
+ unset($this->fields[$index]);
+ }
+ }
+ $this->fields = array_values($this->fields);
+ if (count($fields) !== count($this->fields)) {
+ throw new Exception(
+ get_class($this) . ': invalid fields specified.'
+ );
+ }
+ }
+ $this->limit = $limit;
+ $this->offset = $offset;
}
/**
@@ -148,7 +186,7 @@ public function next()
public function rewind()
{
$this->logger->debug('Executing query');
- $this->sth = $this->query->getRawStatement();
+ $this->sth = $this->query->getRawStatement($this->limit, $this->offset);
$this->logger->debug(sprintf(
'Raw query string: %s',
$this->sth->queryString
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 121b907d89..71d054df9b 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2,10 +2,14 @@
namespace Rest\Controllers;
+use CCR\Log;
use Configuration\XdmodConfiguration;
+use DataWarehouse\Data\BatchDataset;
+use DataWarehouse\Export\RealmManager;
use DataWarehouse\Query\Exceptions\AccessDeniedException;
use DataWarehouse\Query\Exceptions\NotFoundException;
use DataWarehouse\Query\Exceptions\BadRequestException;
+use DataWarehouse\Query\Model\WhereCondition;
use Models\Services\Acls;
use Models\Services\Parameters;
use Models\Services\Realms;
@@ -19,6 +23,7 @@
use XDUser;
use DataWarehouse\Access\MetricExplorer;
use DataWarehouse\Access\Usage;
+use stdClass;
/**
* @SuppressWarnings(PHPMD.StaticAccess)
@@ -328,6 +333,11 @@ public function setupRoutes(Application $app, ControllerCollection $controller)
$controller
->get("$root/plots", "$current::getPlots");
+ $controller
+ ->get("$root/raw-data", "$current::getRawData");
+
+ $controller
+ ->get("$root/raw-data/limit", "$current::getRawDataLimit");
}
/**
@@ -730,7 +740,7 @@ public function getRealms(Request $request, Application $app)
$user = $this->authorize($request);
// Get parameters.
- $queryGroup = $this->getStringParam($request, 'querygroup', false, self::_DEFAULT_QUERY_GROUP);
+ $this->getStringParam($request, 'querygroup', false, self::_DEFAULT_QUERY_GROUP);
// Get the realms for the query group and the user's active role.
$realms = Realms::getRealmsForUser($user);
@@ -888,8 +898,6 @@ public function getDimensionValues(Request $request, Application $app, $dimensio
{
$user = $this->authorize($request);
- $action = 'dimensionValues';
-
// Get parameters.
$offset = $this->getIntParam($request, 'offset', false, 0);
$limit = $this->getIntParam($request, 'limit');
@@ -2181,4 +2189,188 @@ private function getUserStore(\XDUser $user, $realm)
$container = implode('-', array_filter(array(self::_HISTORY_STORE, strtoupper($realm))));
return new \UserStorage($user, $container);
}
+
+ public function getRawData(Request $request, Application $app) {
+ $user = $this->authenticateToken($request);
+ $params = $this->validateRawDataParams($request, $user);
+ $query = $this->getRawDataQuery($params);
+ $logger = $this->getRawDataLogger();
+ $limit = intval(\xd_utilities\getConfiguration(
+ 'datawarehouse',
+ 'rest_raw_row_limit'
+ ));
+ try {
+ $dataset = new BatchDataset(
+ $query,
+ $user,
+ $logger,
+ $params['fields'],
+ $limit,
+ $params['offset']
+ );
+ } catch (\Exception $e) {
+ if (preg_match('/fields/', $e->getMessage())) {
+ throw new BadRequestException('Invalid fields.');
+ } else {
+ throw $e;
+ }
+ }
+ $data = $this->parseRawDataBatchDataset($dataset);
+ return $app->json(array(
+ 'success' => true,
+ 'fields' => $dataset->getHeader(),
+ 'data' => $data
+ ));
+ }
+
+ public function getRawDataLimit(Request $request, Application $app) {
+ $this->authenticateToken($request);
+ $limit = intval(\xd_utilities\getConfiguration(
+ 'datawarehouse',
+ 'rest_raw_row_limit'
+ ));
+ return $app->json(array(
+ 'success' => true,
+ 'data' => $limit
+ ));
+ }
+
+ private function validateRawDataParams($request, $user) {
+ $params = array();
+ list(
+ $params['start_date'], $params['end_date']
+ ) = $this->validateRawDataDateParams($request);
+ $params['realm'] = $this->getStringParam($request, 'realm', true);
+ $queryDescripters = Acls::getQueryDescripters($user, $params['realm']);
+ if (empty($queryDescripters)) {
+ throw new BadRequestException('Invalid realm.');
+ }
+ $params['fields'] = $this->validateRawDataFieldsParam($request);
+ $params['filters'] = $this->validateRawDataFiltersParams(
+ $request,
+ $queryDescripters
+ );
+ $params['offset'] = $this->getIntParam($request, 'offset', false, 0);
+ if ($params['offset'] < 0) {
+ throw new BadRequestException('Offset must be non-negative.');
+ }
+ return $params;
+ }
+
+ private function getRawDataQuery($params) {
+ $realmManager = new RealmManager();
+ $className = $realmManager->getRawDataQueryClass($params['realm']);
+ $query = new $className(
+ array(
+ 'start_date' => $params['start_date'],
+ 'end_date' => $params['end_date']
+ ),
+ 'batch'
+ );
+ $query = $this->setRawDataQueryFilters($query, $params);
+ if ($params['realm'] === "SUPREMM") {
+ $query->addWhereCondition(
+ new WhereCondition('jf.cpu_user', 'IS NOT', 'NULL')
+ );
+ }
+ return $query;
+ }
+
+ private function getRawDataLogger() {
+ // TODO — where to actually log?
+ return Log::factory(
+ 'data-warehouse-raw-data',
+ array(
+ 'console' => false,
+ 'db' => false,
+ 'file' => false,
+ 'mail' => false
+ )
+ );
+ }
+
+ private function parseRawDataBatchDataset($dataset) {
+ $data = array();
+ foreach ($dataset as $record) {
+ $data[] = $record;
+ }
+ return $data;
+ }
+
+ private function validateRawDataDateParams($request) {
+ $startDate = $this->getDateFromISO8601Param(
+ $request,
+ 'start_date',
+ true
+ );
+ $endDate = $this->getDateFromISO8601Param(
+ $request,
+ 'end_date',
+ true
+ );
+ if ($endDate < $startDate) {
+ throw new BadRequestException(
+ 'End date cannot be less than start date.'
+ );
+ }
+ return array($startDate->format('Y-m-d'), $endDate->format('Y-m-d'));
+ }
+
+ private function validateRawDataFieldsParam($request) {
+ $fields = null;
+ $fieldsStr = $this->getStringParam($request, 'fields', false);
+ if ($fieldsStr !== null) {
+ $fields = explode(',', $fieldsStr);
+ }
+ return $fields;
+ }
+
+ private function validateRawDataFiltersParams($request, $queryDescripters) {
+ $filters = null;
+ $filtersParam = $request->get('filters');
+ if ($filtersParam !== null) {
+ $filters = array();
+ foreach ($filtersParam as $filterKey => $filterValuesStr) {
+ $filters[$filterKey] = $this->validateRawDataFilterParam(
+ $queryDescripters,
+ $filterKey,
+ $filterValuesStr
+ );
+ }
+ }
+ return $filters;
+ }
+
+ private function setRawDataQueryFilters($query, $params) {
+ if (count($params['filters']) > 0) {
+ $f = new stdClass();
+ $f->{'data'} = array();
+ foreach ($params['filters'] as $dimension => $values) {
+ foreach ($values as $value) {
+ $f->{'data'}[] = (object) array(
+ 'id' => "$dimension=$value",
+ 'value_id' => $value,
+ 'dimension_id' => $dimension,
+ 'checked' => 1,
+ );
+ }
+ }
+ $query->setFilters($f);
+ }
+ return $query;
+ }
+
+ private function validateRawDataFilterParam(
+ $queryDescripters,
+ $filterKey,
+ $filterValuesStr
+ ) {
+ if (!in_array($filterKey, array_keys($queryDescripters))) {
+ throw new BadRequestException(
+ 'Invalid filter key \'' . $filterKey . '\'.'
+ );
+ }
+ $filterValuesArray = explode(',', $filterValuesStr);
+ return $filterValuesArray;
+ }
}
diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini
index b4f572bf50..ed7311470c 100644
--- a/configuration/portal_settings.ini
+++ b/configuration/portal_settings.ini
@@ -133,6 +133,7 @@ user = "xdmod"
pass = ""
database = "modw"
rest_logfile = "rest_datawarehouse.log"
+rest_raw_row_limit = "10000"
[shredder]
db_engine = "MySQLDB"
diff --git a/docs/xdmod-rest-schema.json b/docs/xdmod-rest-schema.json
index d90373ec38..fa43f81057 100644
--- a/docs/xdmod-rest-schema.json
+++ b/docs/xdmod-rest-schema.json
@@ -30,7 +30,7 @@
"offset": {
"name": "offset",
"in": "query",
- "description": "The start offset for the data in the result set",
+ "description": "The starting offset for the data in the result set",
"schema": {
"type": "integer",
"format": "int32"
@@ -52,7 +52,8 @@
"description": "The data realm to query",
"schema": {
"type": "string"
- }
+ },
+ "example": "Jobs"
},
"dimension": {
"name": "dimension",
@@ -62,6 +63,26 @@
"schema": {
"type": "string"
}
+ },
+ "start_date": {
+ "name": "start_date",
+ "in": "query",
+ "required": true,
+ "description": "Start date of requested data",
+ "schema": {
+ "type": "string",
+ "format": "date"
+ }
+ },
+ "end_date": {
+ "name": "end_date",
+ "in": "query",
+ "required": true,
+ "description": "End date of requested data",
+ "schema": {
+ "type": "string",
+ "format": "date"
+ }
}
},
"schemas": {
@@ -517,25 +538,85 @@
}
},
"securitySchemes": {
- "cookie": {
+ "cookie_xdmod_token": {
"type": "apiKey",
"description": "API token via a cookie",
"name": "xdmod_token",
"in": "cookie"
},
- "token": {
+ "query_param_token": {
"type": "apiKey",
"description": "API token via a query parameter",
"name": "token",
"in": "query"
+ },
+ "http_bearer": {
+ "type": "http",
+ "scheme": "bearer",
+ "description": "API token via HTTP header `Authorization: Bearer `"
+ },
+ "query_param_bearer": {
+ "type": "apiKey",
+ "in": "query",
+ "name": "Bearer",
+ "description": "API token via a query parameter or POST field"
+ }
+ },
+ "examples": {
+ "empty-token": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "No Token Provided.",
+ "code": 0
+ }
+ },
+ "malformed-token": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid token format.",
+ "code": 0
+ }
+ },
+ "invalid-token": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid API token.",
+ "code": 0
+ }
+ },
+ "expired-token": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "The API Token has expired.",
+ "code": 0
+ }
}
}
},
- "security": [{
- "cookie": []
- }, {
- "token": []
- }],
+ "security": [
+ { "cookie_xdmod_token": [] },
+ { "query_param_token": [] }
+ ],
"paths": {
"/controllers/user_interface.php": {
"post": {
@@ -2731,6 +2812,368 @@
}
}
},
+ "/warehouse/raw-data": {
+ "get": {
+ "tags": [
+ "Data Warehouse",
+ "Data Analytics Framework"
+ ],
+ "summary": "Get raw data from the data warehouse",
+ "description": "This will return rows of raw data up to a maximum number of rows. The limit on number of rows can be obtained via `GET /warehouse/raw-data/limit`.",
+ "operationId": "get-warehouse-raw-data",
+ "security": [
+ { "http_bearer": [] },
+ { "query_param_bearer": [] }
+ ],
+ "parameters": [
+ { "$ref": "#/components/parameters/start_date" },
+ { "$ref": "#/components/parameters/end_date" },
+ { "$ref": "#/components/parameters/realm" },
+ {
+ "in": "query",
+ "name": "fields",
+ "required": false,
+ "description": "Identifiers of fields to get",
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [ "Nodes", "Wall Time" ]
+ },
+ "explode": false
+ },
+ {
+ "in": "query",
+ "name": "filters",
+ "required": false,
+ "description": "A mapping of dimension identifiers to their possible values. Results will only be included whose values for each of the given dimensions match one of the corresponding given values.",
+ "style": "deepObject",
+ "schema": {
+ "type": "object",
+ "additionalProperties": {
+ "x-additionalPropertiesName": "dimension identifier",
+ "type": "string",
+ "format": "comma-separated list"
+ }
+ },
+ "example": {
+ "fieldofscience": "177",
+ "provider": "848,856"
+ }
+ },
+ {
+ "in": "query",
+ "name": "offset",
+ "required": false,
+ "description": "Starting row index",
+ "schema": {
+ "type": "integer",
+ "default": 0,
+ "minimum": 0
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "title": "success",
+ "properties": {
+ "success": {
+ "type": "boolean",
+ "enum": [ true ]
+ },
+ "fields": {
+ "type": "array",
+ "description": "Labels of fields returned",
+ "items": {
+ "type": "string"
+ }
+ },
+ "data": {
+ "type": "array",
+ "description": "Rows of raw data",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "nullable": true
+ }
+ }
+ }
+ },
+ "required": [
+ "success",
+ "fields",
+ "data"
+ ]
+ },
+ "examples": {
+ "success": {
+ "description": "Parameters: `start_date=2022-01-01&end_date=2022-01-01&realm=SUPREMM&fields=nodes,cores`",
+ "value": {
+ "success": true,
+ "fields": [ "Cores", "Nodes" ],
+ "data": [ ["1", "1"], ["5", "1"], ["2304", "24"] ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "400": {
+ "description": "Bad Request",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/warehouse-error"
+ },
+ "examples": {
+ "no-start-date": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "start_date is a required parameter",
+ "code": 0
+ }
+ },
+ "no-end-date": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "end_date is a required parameter",
+ "code": 0
+ }
+ },
+ "start-date-malformed": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid value for start_date. Must be a(n) ISO 8601 Date.",
+ "code": 0
+ }
+ },
+ "no-realm": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "realm is a required parameter.",
+ "code": 0
+ }
+ },
+ "end-date-malformed": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid value for end_date. Must be a(n) ISO 8601 Date.",
+ "code": 0
+ }
+ },
+ "end-before-start": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "End date cannot be less than start date.",
+ "code": 0
+ }
+ },
+ "invalid-realm": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid realm.",
+ "code": 104
+ }
+ },
+ "invalid-fields": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid fields.",
+ "code": 104
+ }
+ },
+ "invalid-filter-key": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Invalid filter key 'foo'.",
+ "code": 104
+ }
+ },
+ "negative-offset": {
+ "value": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Offset must be non-negative.",
+ "code": 104
+ }
+ }
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Unauthorized",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/warehouse-error"
+ },
+ "examples": {
+ "empty-token": {
+ "$ref": "#/components/examples/empty-token"
+ },
+ "malformed-token": {
+ "$ref": "#/components/examples/malformed-token"
+ },
+ "invalid-token": {
+ "$ref": "#/components/examples/invalid-token"
+ },
+ "expired-token": {
+ "$ref": "#/components/examples/expired-token"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/warehouse/raw-data/limit": {
+ "get": {
+ "tags": [
+ "Data Warehouse",
+ "Data Analytics Framework"
+ ],
+ "summary": "Get the raw data response limit",
+ "description": "This gets the maximum number of rows that will be returned in a single response from `GET /warehouse/raw-data`.",
+ "operationId": "get-warehouse-raw-data-limit",
+ "security": [
+ { "http_bearer": [] },
+ { "query_param_bearer": [] }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "title": "success",
+ "properties": {
+ "success": {
+ "type": "boolean",
+ "enum": [ true ]
+ },
+ "data": {
+ "type": "integer",
+ "minimum": 0
+ }
+ },
+ "required": [
+ "success",
+ "data"
+ ]
+ },
+ "example": {
+ "success": true,
+ "data": 10000
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Unauthorized",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/warehouse-error"
+ },
+ "examples": {
+ "empty-token": {
+ "$ref": "#/components/examples/empty-token"
+ },
+ "malformed-token": {
+ "$ref": "#/components/examples/malformed-token"
+ },
+ "invalid-token": {
+ "$ref": "#/components/examples/invalid-token"
+ },
+ "expired-token": {
+ "$ref": "#/components/examples/expired-token"
+ }
+ }
+ }
+ }
+ },
+ "500": {
+ "description": "Internal Server Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/warehouse-error"
+ },
+ "example": {
+ "success": false,
+ "count": 0,
+ "total": 0,
+ "totalCount": 0,
+ "results": [],
+ "data": [],
+ "message": "Option 'rest_raw_row_limit' does not exist in section 'datawarehouse'",
+ "code": 0
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/warehouse/export/realms": {
"get": {
"tags": [
@@ -2826,35 +3269,9 @@
"summary": "Create a data export request",
"operationId": "post-warehouse-export-request",
"parameters": [
- {
- "name": "realm",
- "in": "query",
- "required": true,
- "description": "Realm identifier",
- "schema": {
- "type": "string"
- }
- },
- {
- "name": "start_date",
- "in": "query",
- "required": true,
- "description": "Start date of requested data",
- "schema": {
- "type": "string",
- "format": "date"
- }
- },
- {
- "name": "end_date",
- "in": "query",
- "required": true,
- "description": "End date of requested data",
- "schema": {
- "type": "string",
- "format": "date"
- }
- },
+ { "$ref": "#/components/parameters/realm" },
+ { "$ref": "#/components/parameters/start_date" },
+ { "$ref": "#/components/parameters/end_date" },
{
"name": "format",
"in": "query",
diff --git a/tests/artifacts/xdmod/integration/base/error104.json b/tests/artifacts/xdmod/integration/base/error104.json
new file mode 100644
index 0000000000..f8b2150f39
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/base/error104.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "error.json",
+ "code": 104
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json
new file mode 100644
index 0000000000..de94890628
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error104.json",
+ "message": "End date cannot be less than start date."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json
new file mode 100644
index 0000000000..50370cd706
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "Invalid value for end_date. Must be a(n) ISO 8601 Date."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json
new file mode 100644
index 0000000000..39ce2137e8
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error104.json",
+ "message": "Invalid fields."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json
new file mode 100644
index 0000000000..647b8b1a77
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error104.json",
+ "message": "Invalid filter key 'asdf'."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json
new file mode 100644
index 0000000000..ba0ffdf986
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error104.json",
+ "message": "Invalid realm."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json
new file mode 100644
index 0000000000..8a7ad8f3a8
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json
@@ -0,0 +1,4 @@
+{
+ "success": true,
+ "data": 10000
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json
new file mode 100644
index 0000000000..1acf5382bb
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error104.json",
+ "message": "Offset must be non-negative."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json
new file mode 100644
index 0000000000..171624b3d3
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "end_date is a required parameter."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json
new file mode 100644
index 0000000000..2b88e4f665
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "realm is a required parameter."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json
new file mode 100644
index 0000000000..614a1dcebe
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "start_date is a required parameter."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json
new file mode 100644
index 0000000000..1e9722574c
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json
@@ -0,0 +1,4 @@
+{
+ "$extends": "${INTEGRATION_ROOT}/base/error.json",
+ "message": "Invalid value for start_date. Must be a(n) ISO 8601 Date."
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json
new file mode 100644
index 0000000000..475a0595da
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json
@@ -0,0 +1,30 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "success": { "$ref": "#/$defs/success" },
+ "fields": { "$ref": "#/$defs/fields" },
+ "data": { "$ref": "#/$defs/data" }
+ },
+ "required": [ "success", "fields", "data" ],
+ "additionalProperties": false,
+ "$defs": {
+ "success": {
+ "type": "boolean",
+ "enum": [ true ]
+ },
+ "fields": {
+ "type": "array",
+ "items": { "type": "string" },
+ "uniqueItems": true
+ },
+ "data": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/data_row" }
+ },
+ "data_row": {
+ "type": "array",
+ "items": { "type": "string" }
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json
new file mode 100644
index 0000000000..3c3432235a
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json
@@ -0,0 +1,14 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "success_all_fields.spec.json" },
+ {
+ "properties": {
+ "data": {
+ "minItems": 10000,
+ "maxItems": 10000
+ }
+ }
+ }
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json
new file mode 100644
index 0000000000..456fa0aae4
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json
@@ -0,0 +1,14 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "success_all_fields.spec.json" },
+ {
+ "properties": {
+ "data": {
+ "minItems": 66,
+ "maxItems": 66
+ }
+ }
+ }
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json
new file mode 100644
index 0000000000..93013058b7
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json
@@ -0,0 +1,20 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "success.spec.json" },
+ {
+ "properties": {
+ "fields": {
+ "minItems": 28,
+ "maxItems": 28
+ },
+ "data": {
+ "items": {
+ "minItems": 28,
+ "maxItems": 28
+ }
+ }
+ }
+ }
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json
new file mode 100644
index 0000000000..17e0efe8f6
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json
@@ -0,0 +1,25 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "success.spec.json" },
+ {
+ "properties": {
+ "fields": {
+ "items": {
+ "enum": [ "Nodes", "Wall Time" ]
+ },
+ "minItems": 2,
+ "maxItems": 2
+ },
+ "data": {
+ "items": {
+ "minItems": 2,
+ "maxItems": 2
+ },
+ "minItems": 29,
+ "maxItems": 29
+ }
+ }
+ }
+ ]
+}
diff --git a/tests/integration/lib/Rest/WarehouseControllerProviderTest.php b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
index 62549f75d6..721d9dd119 100644
--- a/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
+++ b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
@@ -3,26 +3,17 @@
namespace IntegrationTests\Rest;
use IntegrationTests\BaseTest;
+use TestHarness\TokenHelper;
+use TestHarness\XdmodTestHelper;
class WarehouseControllerProviderTest extends BaseTest
{
- protected static $helpers = array();
+ private static $helper;
public static function setUpBeforeClass()
{
- foreach (array('pub', 'cd') as $user) {
- self::$helpers[$user] = new \TestHarness\XdmodTestHelper();
- if ($user != 'pub') {
- self::$helpers[$user]->authenticate($user);
- }
- }
- }
-
- public static function tearDownAfterClass()
- {
- foreach (self::$helpers as $helper) {
- $helper->logout();
- }
+ parent::setUpBeforeClass();
+ self::$helper = new XdmodTestHelper();
}
/**
@@ -85,10 +76,12 @@ public function aggregateDataAccessControlsProvider()
*/
public function testGetAggregateDataMalformedRequests($user, $params)
{
- $response = self::$helpers[$user]->get('rest/warehouse/aggregatedata', $params);
+ self::$helper->authenticate($user);
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals(400, $response[1]['http_code']);
$this->assertFalse($response[0]['success']);
+ self::$helper->logout();
}
/**
@@ -96,27 +89,33 @@ public function testGetAggregateDataMalformedRequests($user, $params)
*/
public function testGetAggregateDataAccessControls($user, $http_code, $params)
{
- $response = self::$helpers[$user]->get('rest/warehouse/aggregatedata', $params);
+ if ('pub' !== $user) {
+ self::$helper->authenticate($user);
+ }
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals($http_code, $response[1]['http_code']);
$this->assertFalse($response[0]['success']);
+ self::$helper->logout();
}
public function testGetAggregateData()
{
//TODO: Needs further integration for other realms.
- if (!in_array("jobs", self::$XDMOD_REALMS)) {
+ if (!in_array("jobs", parent::$XDMOD_REALMS)) {
$this->markTestSkipped('Needs realm integration.');
}
$params = $this->getAggDataParameterGenerator();
- $response = self::$helpers['cd']->get('rest/warehouse/aggregatedata', $params);
+ self::$helper->authenticate('cd');
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals(200, $response[1]['http_code']);
$this->assertTrue($response[0]['success']);
$this->assertCount($params['limit'], $response[0]['results']);
$this->assertEquals(66, $response[0]['total']);
+ self::$helper->logout();
}
/**
@@ -124,17 +123,214 @@ public function testGetAggregateData()
*/
public function testGetAggregateWithFilters()
{
- if (!in_array("jobs", self::$XDMOD_REALMS)) {
+ if (!in_array("jobs", parent::$XDMOD_REALMS)) {
$this->markTestSkipped('This test requires the Jobs realm');
}
$params = $this->getAggDataParameterGenerator(array('filters' => array('jobsize' => 1)));
- $response = self::$helpers['cd']->get('rest/warehouse/aggregatedata', $params);
+ self::$helper->authenticate('cd');
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals(200, $response[1]['http_code']);
$this->assertTrue($response[0]['success']);
$this->assertCount($params['limit'], $response[0]['results']);
$this->assertEquals(23, $response[0]['total']);
+ self::$helper->logout();
+ }
+
+ /**
+ * @dataProvider provideGetRawDataRoles
+ */
+ public function testGetRawData($role, $tokenTests)
+ {
+ $tokenHelper = new TokenHelper(
+ $this,
+ self::$helper,
+ $role,
+ 'rest/warehouse/raw-data',
+ 'get',
+ null,
+ null,
+ 'rest',
+ 'token_required'
+ );
+ $tokenHelper->runEndpointTests(
+ function ($token) use ($tokenTests, $tokenHelper) {
+ foreach ($tokenTests as $values) {
+ list(
+ $params,
+ $httpCode,
+ $fileName,
+ $validationType
+ ) = $values;
+ $tokenHelper->setParams($params);
+ $tokenHelper->runEndpointTest(
+ $token,
+ $fileName,
+ $httpCode,
+ 'integration/rest/warehouse',
+ $validationType
+ );
+ }
+ }
+ );
+ }
+
+ /**
+ * @dataProvider provideBaseRoles
+ */
+ public function testGetRawDataLimit($role)
+ {
+ $tokenHelper = new TokenHelper(
+ $this,
+ self::$helper,
+ $role,
+ 'rest/warehouse/raw-data/limit',
+ 'get',
+ null,
+ null,
+ 'rest',
+ 'token_required'
+ );
+ $tokenHelper->runEndpointTests(
+ function ($token) use ($tokenHelper) {
+ $tokenHelper->runEndpointTest(
+ $token,
+ 'get_raw_data/limit_success',
+ 200,
+ 'integration/rest/warehouse',
+ 'exact'
+ );
+ }
+ );
+ }
+
+ public function provideGetRawDataRoles()
+ {
+ $tokenTests = array(
+ array(null, 400, 'get_raw_data/no_start_date', 'exact'),
+ array(
+ array('start_date' => '2017-01-01'),
+ 400,
+ 'get_raw_data/no_end_date',
+ 'exact'
+ ),
+ array(
+ array('start_date' => '2017'),
+ 400,
+ 'get_raw_data/start_date_malformed',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01'
+ ),
+ 400,
+ 'get_raw_data/no_realm',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017'
+ ),
+ 400,
+ 'get_raw_data/end_date_malformed',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2016-01-01'
+ ),
+ 400,
+ 'get_raw_data/end_before_start',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'asdf'
+ ),
+ 400,
+ 'get_raw_data/invalid_realm',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'Jobs',
+ 'fields' => 'asdf'
+ ),
+ 400,
+ 'get_raw_data/invalid_fields',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'Jobs',
+ 'fields' => 'Nodes',
+ 'filters[asdf]' => 177
+ ),
+ 400,
+ 'get_raw_data/invalid_filter_key',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'Jobs',
+ 'offset' => '-1'
+ ),
+ 400,
+ 'get_raw_data/negative_offset',
+ 'exact'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'Jobs'
+ ),
+ 200,
+ 'get_raw_data/success_0.spec',
+ 'schema'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'Jobs',
+ 'offset' => '16500'
+ ),
+ 200,
+ 'get_raw_data/success_16500.spec',
+ 'schema'
+ ),
+ array(
+ array(
+ 'start_date' => '2017-01-01',
+ 'end_date' => '2017-01-01',
+ 'realm' => 'Jobs',
+ 'fields' => 'Nodes,Wall Time',
+ 'filters[resource]' => '1,2',
+ 'filters[fieldofscience]' => '10,91'
+ ),
+ 200,
+ 'get_raw_data/success_fields_and_filters.spec',
+ 'schema'
+ )
+ );
+ return array(
+ array('pub', $tokenTests),
+ array('usr', $tokenTests)
+ );
}
}
diff --git a/tests/integration/lib/TestHarness/TokenHelper.php b/tests/integration/lib/TestHarness/TokenHelper.php
index 463d4e2eb0..11c57ec9d2 100644
--- a/tests/integration/lib/TestHarness/TokenHelper.php
+++ b/tests/integration/lib/TestHarness/TokenHelper.php
@@ -186,6 +186,11 @@ public static function expireToken($userId)
return $db->execute($query, $params) === 1;
}
+ public function setParams($params)
+ {
+ $this->params = $params;
+ }
+
private function setExpectedErrorOutput(
$type,
$fileName = null,
From 594314075fceb2aa779b7107a8b3aa31d1aecc24 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Thu, 20 Apr 2023 13:50:24 -0400
Subject: [PATCH 03/21] Add `rest_raw_row_limit` setting to template.
---
templates/portal_settings.template | 1 +
1 file changed, 1 insertion(+)
diff --git a/templates/portal_settings.template b/templates/portal_settings.template
index c358886879..68c7b37491 100644
--- a/templates/portal_settings.template
+++ b/templates/portal_settings.template
@@ -133,6 +133,7 @@ user = "[:datawarehouse_user:]"
pass = "[:datawarehouse_pass:]"
database = "modw"
rest_logfile = "rest_datawarehouse.log"
+rest_raw_row_limit = "10000"
[shredder]
db_engine = "MySQLDB"
From a08f66e8cb0526d7df299d04ffc5678f88d4e66a Mon Sep 17 00:00:00 2001
From: Aaron Weeden
Date: Sat, 22 Apr 2023 10:34:57 -0400
Subject: [PATCH 04/21] Use is_null to check for null values.
---
classes/Rest/Controllers/WarehouseControllerProvider.php | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 71d054df9b..5195524ae0 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2319,7 +2319,7 @@ private function validateRawDataDateParams($request) {
private function validateRawDataFieldsParam($request) {
$fields = null;
$fieldsStr = $this->getStringParam($request, 'fields', false);
- if ($fieldsStr !== null) {
+ if (!is_null($fieldsStr)) {
$fields = explode(',', $fieldsStr);
}
return $fields;
@@ -2328,7 +2328,7 @@ private function validateRawDataFieldsParam($request) {
private function validateRawDataFiltersParams($request, $queryDescripters) {
$filters = null;
$filtersParam = $request->get('filters');
- if ($filtersParam !== null) {
+ if (!is_null($filtersParam)) {
$filters = array();
foreach ($filtersParam as $filterKey => $filterValuesStr) {
$filters[$filterKey] = $this->validateRawDataFilterParam(
From 7360d3257c9f7a823f49059a0bf59a57875ec12b Mon Sep 17 00:00:00 2001
From: Aaron Weeden
Date: Sat, 22 Apr 2023 10:38:13 -0400
Subject: [PATCH 05/21] Prefer yoda condition to see literal faster.
---
classes/Rest/Controllers/WarehouseControllerProvider.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 5195524ae0..5d3ec142f6 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2268,7 +2268,7 @@ private function getRawDataQuery($params) {
'batch'
);
$query = $this->setRawDataQueryFilters($query, $params);
- if ($params['realm'] === "SUPREMM") {
+ if ('SUPREMM' === $params['realm']) {
$query->addWhereCondition(
new WhereCondition('jf.cpu_user', 'IS NOT', 'NULL')
);
From 805bb493f57661bda17ed3c48854e86647530d39 Mon Sep 17 00:00:00 2001
From: Aaron Weeden
Date: Sat, 22 Apr 2023 10:44:13 -0400
Subject: [PATCH 06/21] Use function for duplicated code.
---
.../Controllers/WarehouseControllerProvider.php | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 5d3ec142f6..5abbc4ac82 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2195,10 +2195,7 @@ public function getRawData(Request $request, Application $app) {
$params = $this->validateRawDataParams($request, $user);
$query = $this->getRawDataQuery($params);
$logger = $this->getRawDataLogger();
- $limit = intval(\xd_utilities\getConfiguration(
- 'datawarehouse',
- 'rest_raw_row_limit'
- ));
+ $limit = $this->getConfiguredRawDataLimit();
try {
$dataset = new BatchDataset(
$query,
@@ -2225,10 +2222,7 @@ public function getRawData(Request $request, Application $app) {
public function getRawDataLimit(Request $request, Application $app) {
$this->authenticateToken($request);
- $limit = intval(\xd_utilities\getConfiguration(
- 'datawarehouse',
- 'rest_raw_row_limit'
- ));
+ $limit = $this->getConfiguredRawDataLimit();
return $app->json(array(
'success' => true,
'data' => $limit
@@ -2289,6 +2283,13 @@ private function getRawDataLogger() {
);
}
+ private function getConfiguredRawDataLimit() {
+ return intval(\xd_utilities\getConfiguration(
+ 'datawarehouse',
+ 'rest_raw_row_limit'
+ ));
+ }
+
private function parseRawDataBatchDataset($dataset) {
$data = array();
foreach ($dataset as $record) {
From e8e1c95c2151c3066dd6e0cd2712cebe77a938bc Mon Sep 17 00:00:00 2001
From: Aaron Weeden
Date: Sat, 22 Apr 2023 11:58:34 -0400
Subject: [PATCH 07/21] Use consistent brace style.
---
.../WarehouseControllerProvider.php | 36 ++++++++++++-------
1 file changed, 24 insertions(+), 12 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 5abbc4ac82..8b7d2686bb 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -1844,7 +1844,8 @@ private function processHistoryDefaultRealmRequest(Application $app, XDUser $use
);
}
- private function encodeFloatArray(array $in) {
+ private function encodeFloatArray(array $in)
+ {
$out = array();
foreach ($in as $key => $value) {
if (is_float($value) && is_nan($value)) {
@@ -2190,7 +2191,8 @@ private function getUserStore(\XDUser $user, $realm)
return new \UserStorage($user, $container);
}
- public function getRawData(Request $request, Application $app) {
+ public function getRawData(Request $request, Application $app)
+ {
$user = $this->authenticateToken($request);
$params = $this->validateRawDataParams($request, $user);
$query = $this->getRawDataQuery($params);
@@ -2220,7 +2222,8 @@ public function getRawData(Request $request, Application $app) {
));
}
- public function getRawDataLimit(Request $request, Application $app) {
+ public function getRawDataLimit(Request $request, Application $app)
+ {
$this->authenticateToken($request);
$limit = $this->getConfiguredRawDataLimit();
return $app->json(array(
@@ -2229,7 +2232,8 @@ public function getRawDataLimit(Request $request, Application $app) {
));
}
- private function validateRawDataParams($request, $user) {
+ private function validateRawDataParams($request, $user)
+ {
$params = array();
list(
$params['start_date'], $params['end_date']
@@ -2251,7 +2255,8 @@ private function validateRawDataParams($request, $user) {
return $params;
}
- private function getRawDataQuery($params) {
+ private function getRawDataQuery($params)
+ {
$realmManager = new RealmManager();
$className = $realmManager->getRawDataQueryClass($params['realm']);
$query = new $className(
@@ -2270,7 +2275,8 @@ private function getRawDataQuery($params) {
return $query;
}
- private function getRawDataLogger() {
+ private function getRawDataLogger()
+ {
// TODO — where to actually log?
return Log::factory(
'data-warehouse-raw-data',
@@ -2283,14 +2289,16 @@ private function getRawDataLogger() {
);
}
- private function getConfiguredRawDataLimit() {
+ private function getConfiguredRawDataLimit()
+ {
return intval(\xd_utilities\getConfiguration(
'datawarehouse',
'rest_raw_row_limit'
));
}
- private function parseRawDataBatchDataset($dataset) {
+ private function parseRawDataBatchDataset($dataset)
+ {
$data = array();
foreach ($dataset as $record) {
$data[] = $record;
@@ -2298,7 +2306,8 @@ private function parseRawDataBatchDataset($dataset) {
return $data;
}
- private function validateRawDataDateParams($request) {
+ private function validateRawDataDateParams($request)
+ {
$startDate = $this->getDateFromISO8601Param(
$request,
'start_date',
@@ -2317,7 +2326,8 @@ private function validateRawDataDateParams($request) {
return array($startDate->format('Y-m-d'), $endDate->format('Y-m-d'));
}
- private function validateRawDataFieldsParam($request) {
+ private function validateRawDataFieldsParam($request)
+ {
$fields = null;
$fieldsStr = $this->getStringParam($request, 'fields', false);
if (!is_null($fieldsStr)) {
@@ -2326,7 +2336,8 @@ private function validateRawDataFieldsParam($request) {
return $fields;
}
- private function validateRawDataFiltersParams($request, $queryDescripters) {
+ private function validateRawDataFiltersParams($request, $queryDescripters)
+ {
$filters = null;
$filtersParam = $request->get('filters');
if (!is_null($filtersParam)) {
@@ -2342,7 +2353,8 @@ private function validateRawDataFiltersParams($request, $queryDescripters) {
return $filters;
}
- private function setRawDataQueryFilters($query, $params) {
+ private function setRawDataQueryFilters($query, $params)
+ {
if (count($params['filters']) > 0) {
$f = new stdClass();
$f->{'data'} = array();
From c324fc5b12852bd158a7d0d42181c68d3ebe6f25 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Mon, 24 Apr 2023 10:45:16 -0400
Subject: [PATCH 08/21] Set logging to database.
---
classes/Rest/Controllers/WarehouseControllerProvider.php | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 8b7d2686bb..0a9f2ba601 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2277,12 +2277,10 @@ private function getRawDataQuery($params)
private function getRawDataLogger()
{
- // TODO — where to actually log?
return Log::factory(
- 'data-warehouse-raw-data',
+ 'data-warehouse-raw-data-rest',
array(
'console' => false,
- 'db' => false,
'file' => false,
'mail' => false
)
From 16c693e267c611a66a82621eedfa01a8e5599fd3 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Fri, 5 May 2023 13:56:49 -0400
Subject: [PATCH 09/21] Restore functionality overwritten by merge.
---
.../xdmod/integration/base/date.spec.json | 5 -
.../xdmod/integration/base/error.json | 10 --
.../xdmod/integration/base/error104.json | 4 -
.../xdmod/integration/base/success.spec.json | 13 --
.../output/get_dw_descripter.spec.json | 152 ------------------
.../output/authentication_error.json | 4 -
.../user/api_token/output/create_failure.json | 4 -
.../api_token/output/create_success.spec.json | 24 ---
.../user/api_token/output/empty_token.json | 4 -
.../user/api_token/output/expired_token.json | 4 -
.../api_token/output/get_success.spec.json | 23 ---
.../user/api_token/output/invalid_token.json | 4 -
.../api_token/output/malformed_token.json | 4 -
.../rest/user/api_token/output/not_found.json | 4 -
.../user/api_token/output/revoke_success.json | 4 -
.../api_token/output/session_expired.json | 5 -
.../export/output/get_realms.spec.json | 100 ------------
.../rest/warehouse/input/get_raw_data.json | 131 +++++++++++++++
.../warehouse/input/get_raw_data_limit.json | 8 +
.../output/get_raw_data/end_before_start.json | 4 -
.../get_raw_data/end_date_malformed.json | 4 -
.../output/get_raw_data/invalid_fields.json | 4 -
.../get_raw_data/invalid_filter_key.json | 4 -
.../output/get_raw_data/invalid_realm.json | 4 -
.../output/get_raw_data/limit_success.json | 4 -
.../output/get_raw_data/negative_offset.json | 4 -
.../output/get_raw_data/no_end_date.json | 4 -
.../output/get_raw_data/no_realm.json | 4 -
.../output/get_raw_data/no_start_date.json | 4 -
.../get_raw_data/start_date_malformed.json | 4 -
.../output/get_raw_data/success.spec.json | 30 ----
.../output/get_raw_data/success_0.spec.json | 14 --
.../get_raw_data/success_16500.spec.json | 14 --
.../get_raw_data/success_all_fields.spec.json | 20 ---
.../success_fields_and_filters.spec.json | 25 ---
.../Rest/WarehouseControllerProviderTest.php | 77 ++++++---
36 files changed, 194 insertions(+), 537 deletions(-)
delete mode 100644 tests/artifacts/xdmod/integration/base/date.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/base/error.json
delete mode 100644 tests/artifacts/xdmod/integration/base/error104.json
delete mode 100644 tests/artifacts/xdmod/integration/base/success.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data_limit.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json
delete mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json
diff --git a/tests/artifacts/xdmod/integration/base/date.spec.json b/tests/artifacts/xdmod/integration/base/date.spec.json
deleted file mode 100644
index 5c828eab26..0000000000
--- a/tests/artifacts/xdmod/integration/base/date.spec.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "type": "string",
- "pattern": "[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}"
-}
diff --git a/tests/artifacts/xdmod/integration/base/error.json b/tests/artifacts/xdmod/integration/base/error.json
deleted file mode 100644
index 4e600b5109..0000000000
--- a/tests/artifacts/xdmod/integration/base/error.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "success": false,
- "count": 0,
- "total": 0,
- "totalCount": 0,
- "results": [],
- "data": [],
- "message": "",
- "code": 0
-}
diff --git a/tests/artifacts/xdmod/integration/base/error104.json b/tests/artifacts/xdmod/integration/base/error104.json
deleted file mode 100644
index f8b2150f39..0000000000
--- a/tests/artifacts/xdmod/integration/base/error104.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "error.json",
- "code": 104
-}
diff --git a/tests/artifacts/xdmod/integration/base/success.spec.json b/tests/artifacts/xdmod/integration/base/success.spec.json
deleted file mode 100644
index f89fb09e2f..0000000000
--- a/tests/artifacts/xdmod/integration/base/success.spec.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema",
- "type": "object",
- "properties": {
- "success": {
- "type": "boolean",
- "enum": [ true ]
- }
- },
- "required": [
- "success"
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json b/tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json
deleted file mode 100644
index 998af806bb..0000000000
--- a/tests/artifacts/xdmod/integration/controllers/metric_explorer/output/get_dw_descripter.spec.json
+++ /dev/null
@@ -1,152 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "type": "object",
- "properties": {
- "totalCount": {
- "type": "number",
- "enum": [ 1 ]
- },
- "data": {
- "type": "array",
- "items": { "$ref": "#/$defs/data_item" },
- "minItems": 1,
- "maxItems": 1,
- "uniqueItems": true
- }
- },
- "required": [ "totalCount", "data" ],
- "additionalProperties": false,
- "$defs": {
- "data_item": {
- "type": "object",
- "properties": {
- "realms": { "$ref": "#/$defs/realms" }
- },
- "required": [ "realms" ],
- "additionalProperties": false
- },
- "realms": {
- "type": "object",
- "properties": {
- "Cloud": {
- "allOf": [
- { "$ref": "#/$defs/realm" },
- { "$ref": "#/$defs/cloud_realm" }
- ]
- },
- "Jobs": {
- "allOf" : [
- { "$ref": "#/$defs/realm" },
- { "$ref": "#/$defs/jobs_realm" }
- ]
- },
- "Storage": {
- "allOf" : [
- { "$ref": "#/$defs/realm" },
- { "$ref": "#/$defs/storage_realm" }
- ]
- }
- },
- "required": [ "Cloud", "Jobs", "Storage" ],
- "additionalProperties": false
- },
- "realm": {
- "type": "object",
- "properties": {
- "metrics": { "$ref": "#/$defs/metrics" },
- "dimensions": { "$ref": "#/$defs/dimensions" },
- "text": { "type": "string" },
- "category": { "type": "string" }
- },
- "required": [ "metrics", "dimensions", "text", "category" ],
- "additionalProperties": false
- },
- "cloud_realm": {
- "properties": {
- "metrics": {
- "minProperties": 10,
- "maxProperties": 10
- },
- "dimensions": {
- "minProperties": 14,
- "maxProperties": 14
- },
- "text": {
- "enum": [ "Cloud" ]
- },
- "category": {
- "enum": [ "Cloud" ]
- }
- }
- },
- "jobs_realm": {
- "properties": {
- "metrics": {
- "minProperties": 26,
- "maxProperties": 26
- },
- "dimensions": {
- "minProperties": 16,
- "maxProperties": 16
- },
- "text": {
- "enum": [ "Jobs" ]
- },
- "category": {
- "enum": [ "Jobs" ]
- }
- }
- },
- "storage_realm": {
- "properties": {
- "metrics": {
- "minProperties": 7,
- "maxProperties": 7
- },
- "dimensions": {
- "minProperties": 10,
- "maxProperties": 10
- },
- "text": {
- "enum": [ "Storage" ]
- },
- "category": {
- "enum": [ "Storage" ]
- }
- }
- },
- "metrics": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- { "$ref": "#/$defs/metric_or_dimension" },
- {
- "properties": {
- "std_err": { "type": "boolean" }
- }
- }
- ]
- }
- }
- },
- "dimensions": {
- "type": "object",
- "patternProperties": {
- "^.*$": { "$ref": "#/$defs/metric_or_dimension" }
- }
- },
- "metric_or_dimension": {
- "type": "object",
- "properties": {
- "text": {
- "type": "string"
- },
- "info": {
- "type": "string"
- }
- },
- "required": [ "text", "info" ]
- }
- }
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json
deleted file mode 100644
index f778eee9dc..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/authentication_error.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "An error was encountered while attempting to process the requested authorization procedure."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json
deleted file mode 100644
index 77705326ed..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_failure.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "Token already exists."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json
deleted file mode 100644
index a5e81d10e7..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/create_success.spec.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- { "$ref": "${INTEGRATION_ROOT}/base/success.spec.json" },
- {
- "properties": {
- "data": {
- "type": "object",
- "properties": {
- "token": {
- "type": "string",
- "pattern": "^[0-9]+\\.[0-9a-f]{64}$"
- },
- "expiration_date": {
- "$ref": "${INTEGRATION_ROOT}/base/date.spec.json"
- }
- },
- "required": [ "token", "expiration_date" ]
- }
- },
- "required": [ "data" ]
- }
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json
deleted file mode 100644
index f4be0727bb..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/empty_token.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "No Token Provided."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json
deleted file mode 100644
index 8c028c39d8..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/expired_token.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "The API Token has expired."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json
deleted file mode 100644
index 34cd7b0c39..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/get_success.spec.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- { "$ref": "${INTEGRATION_ROOT}/base/success.spec.json" },
- {
- "properties": {
- "data": {
- "type": "object",
- "properties": {
- "created_on": {
- "$ref": "${INTEGRATION_ROOT}/base/date.spec.json"
- },
- "expiration_date": {
- "$ref": "${INTEGRATION_ROOT}/base/date.spec.json"
- }
- },
- "required": [ "created_on", "expiration_date" ]
- }
- },
- "required": [ "data" ]
- }
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json
deleted file mode 100644
index 6ad39a9032..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/invalid_token.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "Invalid API token."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json
deleted file mode 100644
index a93df9acc9..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/malformed_token.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "Invalid token format."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json
deleted file mode 100644
index 133ce7170e..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/not_found.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "API token not found."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json
deleted file mode 100644
index f918ae563c..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/revoke_success.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "success": true,
- "message": "Token successfully revoked."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json b/tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json
deleted file mode 100644
index a95387487b..0000000000
--- a/tests/artifacts/xdmod/integration/rest/user/api_token/output/session_expired.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "Session Expired",
- "code": 2
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json
deleted file mode 100644
index 3ad5f561be..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/export/output/get_realms.spec.json
+++ /dev/null
@@ -1,100 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- {
- "type": "object",
- "properties": {
- "success": {
- "type": "boolean",
- "enum": [ true ]
- },
- "data": {
- "type": "array",
- "items": { "$ref": "#/$defs/realm" },
- "minItems": 2,
- "maxItems": 2,
- "uniqueItems": true
- },
- "total": {
- "type": "number",
- "enum": [ 2 ]
- }
- },
- "required": [ "success", "data", "total" ],
- "additionalProperties": false
- },
- { "$ref": "#/$defs/jobs_realm" },
- { "$ref": "#/$defs/cloud_realm" }
- ],
- "$defs": {
- "realm": {
- "type": "object",
- "properties": {
- "id": { "type": "string" },
- "name": { "type": "string" },
- "fields": { "$ref": "#/$defs/fields" }
- },
- "required": [ "id", "name", "fields" ],
- "additionalProperties": false
- },
- "jobs_realm": {
- "properties": {
- "data": {
- "contains": {
- "properties": {
- "id": { "enum": [ "Jobs" ] },
- "name": { "enum": [ "Jobs" ] },
- "fields": {
- "minItems": 28,
- "maxItems": 28
- }
- }
- },
- "minContains": 1,
- "maxContains": 1
- }
- }
- },
- "cloud_realm": {
- "properties": {
- "data": {
- "contains": {
- "properties": {
- "id": { "enum": [ "Cloud" ] },
- "name": { "enum": [ "Cloud" ] },
- "fields": {
- "minItems": 16,
- "maxItems": 16
- }
- }
- },
- "minContains": 1,
- "maxContains": 1
- }
- }
- },
- "fields": {
- "type": "array",
- "items": { "$ref": "#/$defs/field" },
- "uniqueItems": true
- },
- "field": {
- "type": "object",
- "properties": {
- "name": { "type": "string" },
- "alias": { "type": "string" },
- "display": { "type": "string" },
- "anonymize": { "type": "boolean" },
- "documentation": { "type": "string" }
- },
- "required": [
- "name",
- "alias",
- "display",
- "anonymize",
- "documentation"
- ],
- "additionalProperties": false
- }
- }
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json b/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json
new file mode 100644
index 0000000000..fbea8011f6
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json
@@ -0,0 +1,131 @@
+{
+ "defaults": {
+ "path": "rest/warehouse/raw-data",
+ "method": "get",
+ "params": null,
+ "data": null,
+ "endpoint_type": "rest",
+ "authentication_type": "token_required"
+ },
+ "start_date_malformed": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017"
+ }
+ }
+ },
+ "no_end_date": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01"
+ }
+ }
+ },
+ "end_date_malformed": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017"
+ }
+ }
+ },
+ "end_before_start": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2016-01-01"
+ }
+ }
+ },
+ "no_realm": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01"
+ }
+ }
+ },
+ "invalid_realm": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "asdf"
+ }
+ }
+ },
+ "invalid_fields": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "Jobs",
+ "fields": "asdf"
+ }
+ }
+ },
+ "invalid_filter_key": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "Jobs",
+ "fields": "Nodes",
+ "filters[asdf]": "177"
+ }
+ }
+ },
+ "negative_offset": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "Jobs",
+ "offset": "-1"
+ }
+ }
+ },
+ "success_0": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "Jobs"
+ }
+ }
+ },
+ "success_16500": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "Jobs",
+ "offset": "16500"
+ }
+ }
+ },
+ "success_fields_and_filters": {
+ "$ref-with-overwrite": "get_raw_data.json#/defaults",
+ "$overwrite": {
+ "params": {
+ "start_date": "2017-01-01",
+ "end_date": "2017-01-01",
+ "realm": "Jobs",
+ "fields": "Nodes,Wall Time",
+ "filters[resource]": "1,2",
+ "filters[fieldofscience]": "10,91"
+ }
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data_limit.json b/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data_limit.json
new file mode 100644
index 0000000000..0a1a1d3a73
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data_limit.json
@@ -0,0 +1,8 @@
+{
+ "path": "rest/warehouse/raw-data/limit",
+ "method": "get",
+ "params": null,
+ "data": null,
+ "endpoint_type": "rest",
+ "authentication_type": "token_required"
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json
deleted file mode 100644
index de94890628..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_before_start.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error104.json",
- "message": "End date cannot be less than start date."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json
deleted file mode 100644
index 50370cd706..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/end_date_malformed.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "Invalid value for end_date. Must be a(n) ISO 8601 Date."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json
deleted file mode 100644
index 39ce2137e8..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_fields.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error104.json",
- "message": "Invalid fields."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json
deleted file mode 100644
index 647b8b1a77..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_filter_key.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error104.json",
- "message": "Invalid filter key 'asdf'."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json
deleted file mode 100644
index ba0ffdf986..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/invalid_realm.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error104.json",
- "message": "Invalid realm."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json
deleted file mode 100644
index 8a7ad8f3a8..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/limit_success.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "success": true,
- "data": 10000
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json
deleted file mode 100644
index 1acf5382bb..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/negative_offset.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error104.json",
- "message": "Offset must be non-negative."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json
deleted file mode 100644
index 171624b3d3..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_end_date.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "end_date is a required parameter."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json
deleted file mode 100644
index 2b88e4f665..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_realm.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "realm is a required parameter."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json
deleted file mode 100644
index 614a1dcebe..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/no_start_date.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "start_date is a required parameter."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json
deleted file mode 100644
index 1e9722574c..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/start_date_malformed.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "$extends": "${INTEGRATION_ROOT}/base/error.json",
- "message": "Invalid value for start_date. Must be a(n) ISO 8601 Date."
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json
deleted file mode 100644
index 475a0595da..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success.spec.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "type": "object",
- "properties": {
- "success": { "$ref": "#/$defs/success" },
- "fields": { "$ref": "#/$defs/fields" },
- "data": { "$ref": "#/$defs/data" }
- },
- "required": [ "success", "fields", "data" ],
- "additionalProperties": false,
- "$defs": {
- "success": {
- "type": "boolean",
- "enum": [ true ]
- },
- "fields": {
- "type": "array",
- "items": { "type": "string" },
- "uniqueItems": true
- },
- "data": {
- "type": "array",
- "items": { "$ref": "#/$defs/data_row" }
- },
- "data_row": {
- "type": "array",
- "items": { "type": "string" }
- }
- }
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json
deleted file mode 100644
index 3c3432235a..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_0.spec.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- { "$ref": "success_all_fields.spec.json" },
- {
- "properties": {
- "data": {
- "minItems": 10000,
- "maxItems": 10000
- }
- }
- }
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json
deleted file mode 100644
index 456fa0aae4..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_16500.spec.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- { "$ref": "success_all_fields.spec.json" },
- {
- "properties": {
- "data": {
- "minItems": 66,
- "maxItems": 66
- }
- }
- }
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json
deleted file mode 100644
index 93013058b7..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_all_fields.spec.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- { "$ref": "success.spec.json" },
- {
- "properties": {
- "fields": {
- "minItems": 28,
- "maxItems": 28
- },
- "data": {
- "items": {
- "minItems": 28,
- "maxItems": 28
- }
- }
- }
- }
- ]
-}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json
deleted file mode 100644
index 17e0efe8f6..0000000000
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data/success_fields_and_filters.spec.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "allOf": [
- { "$ref": "success.spec.json" },
- {
- "properties": {
- "fields": {
- "items": {
- "enum": [ "Nodes", "Wall Time" ]
- },
- "minItems": 2,
- "maxItems": 2
- },
- "data": {
- "items": {
- "minItems": 2,
- "maxItems": 2
- },
- "minItems": 29,
- "maxItems": 29
- }
- }
- }
- ]
-}
diff --git a/tests/integration/lib/Rest/WarehouseControllerProviderTest.php b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
index 62549f75d6..68a14d66fc 100644
--- a/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
+++ b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
@@ -2,27 +2,17 @@
namespace IntegrationTests\Rest;
-use IntegrationTests\BaseTest;
+use TestHarness\TokenAuthTest;
+use TestHarness\XdmodTestHelper;
-class WarehouseControllerProviderTest extends BaseTest
+class WarehouseControllerProviderTest extends TokenAuthTest
{
- protected static $helpers = array();
+ private static $helper;
public static function setUpBeforeClass()
{
- foreach (array('pub', 'cd') as $user) {
- self::$helpers[$user] = new \TestHarness\XdmodTestHelper();
- if ($user != 'pub') {
- self::$helpers[$user]->authenticate($user);
- }
- }
- }
-
- public static function tearDownAfterClass()
- {
- foreach (self::$helpers as $helper) {
- $helper->logout();
- }
+ parent::setUpBeforeClass();
+ self::$helper = new XdmodTestHelper();
}
/**
@@ -85,10 +75,12 @@ public function aggregateDataAccessControlsProvider()
*/
public function testGetAggregateDataMalformedRequests($user, $params)
{
- $response = self::$helpers[$user]->get('rest/warehouse/aggregatedata', $params);
+ self::$helper->authenticate($user);
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals(400, $response[1]['http_code']);
$this->assertFalse($response[0]['success']);
+ self::$helper->logout();
}
/**
@@ -96,27 +88,33 @@ public function testGetAggregateDataMalformedRequests($user, $params)
*/
public function testGetAggregateDataAccessControls($user, $http_code, $params)
{
- $response = self::$helpers[$user]->get('rest/warehouse/aggregatedata', $params);
+ if ('pub' !== $user) {
+ self::$helper->authenticate($user);
+ }
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals($http_code, $response[1]['http_code']);
$this->assertFalse($response[0]['success']);
+ self::$helper->logout();
}
public function testGetAggregateData()
{
//TODO: Needs further integration for other realms.
- if (!in_array("jobs", self::$XDMOD_REALMS)) {
+ if (!in_array("jobs", parent::$XDMOD_REALMS)) {
$this->markTestSkipped('Needs realm integration.');
}
$params = $this->getAggDataParameterGenerator();
- $response = self::$helpers['cd']->get('rest/warehouse/aggregatedata', $params);
+ self::$helper->authenticate('cd');
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals(200, $response[1]['http_code']);
$this->assertTrue($response[0]['success']);
$this->assertCount($params['limit'], $response[0]['results']);
$this->assertEquals(66, $response[0]['total']);
+ self::$helper->logout();
}
/**
@@ -124,17 +122,52 @@ public function testGetAggregateData()
*/
public function testGetAggregateWithFilters()
{
- if (!in_array("jobs", self::$XDMOD_REALMS)) {
+ if (!in_array("jobs", parent::$XDMOD_REALMS)) {
$this->markTestSkipped('This test requires the Jobs realm');
}
$params = $this->getAggDataParameterGenerator(array('filters' => array('jobsize' => 1)));
- $response = self::$helpers['cd']->get('rest/warehouse/aggregatedata', $params);
+ self::$helper->authenticate('cd');
+ $response = self::$helper->get('rest/warehouse/aggregatedata', $params);
$this->assertEquals(200, $response[1]['http_code']);
$this->assertTrue($response[0]['success']);
$this->assertCount($params['limit'], $response[0]['results']);
$this->assertEquals(23, $response[0]['total']);
+ self::$helper->logout();
+ }
+
+ /**
+ * @dataProvider provideGetRawDataRoles
+ */
+ public function testGetRawData($role)
+ {
+ parent::runTokenAuthTests(
+ $role,
+ 'integration/rest/warehouse',
+ 'get_raw_data'
+ );
+ }
+
+ /**
+ * Only provide one non-public role, since the tests involve pulling a lot
+ * of data and are thus slow.
+ */
+ public function provideGetRawDataRoles()
+ {
+ return [['pub'], ['usr']];
+ }
+
+ /**
+ * @dataProvider provideBaseRoles
+ */
+ public function testGetRawDataLimit($role)
+ {
+ parent::runTokenAuthTests(
+ $role,
+ 'integration/rest/warehouse',
+ 'get_raw_data_limit'
+ );
}
}
From 26b0e52dc70a9c352cd2f14a0fe134796c155d22 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Fri, 5 May 2023 14:51:29 -0400
Subject: [PATCH 10/21] Restore functionality overwritten by merge.
---
.../rest/warehouse/output/get_raw_data.json | 213 ++++++++++++++++++
.../warehouse/output/get_raw_data_limit.json | 7 +
2 files changed, 220 insertions(+)
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json
create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data_limit.json
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json
new file mode 100644
index 0000000000..40ce844fbd
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json
@@ -0,0 +1,213 @@
+{
+ "defaults": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "start_date is a required parameter."
+ }
+ }
+ },
+ "start_date_malformed": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "Invalid value for start_date. Must be a(n) ISO 8601 Date."
+ }
+ }
+ },
+ "no_end_date": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "end_date is a required parameter."
+ }
+ }
+ },
+ "end_date_malformed": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "Invalid value for end_date. Must be a(n) ISO 8601 Date."
+ }
+ }
+ },
+ "end_before_start": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "End date cannot be less than start date.",
+ "code": 104
+ }
+ }
+ },
+ "no_realm": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "realm is a required parameter."
+ }
+ }
+ },
+ "invalid_realm": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "Invalid realm.",
+ "code": 104
+ }
+ }
+ },
+ "invalid_fields": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "Invalid fields.",
+ "code": 104
+ }
+ }
+ },
+ "invalid_filter_key": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "Invalid filter key 'asdf'.",
+ "code": 104
+ }
+ }
+ },
+ "negative_offset": {
+ "status_code": 400,
+ "body": {
+ "$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
+ "$overwrite": {
+ "message": "Offset must be non-negative.",
+ "code": 104
+ }
+ }
+ },
+ "success_body": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "get_raw_data_success_body",
+ "type": "object",
+ "properties": {
+ "success": { "$ref": "#/$defs/success" },
+ "fields": { "$ref": "#/$defs/fields" },
+ "data": { "$ref": "#/$defs/data" }
+ },
+ "required": [ "success", "fields", "data" ],
+ "additionalProperties": false,
+ "$defs": {
+ "success": {
+ "type": "boolean",
+ "enum": [ true ]
+ },
+ "fields": {
+ "type": "array",
+ "items": { "type": "string" },
+ "uniqueItems": true
+ },
+ "data": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/data_row" }
+ },
+ "data_row": {
+ "type": "array",
+ "items": { "type": "string" }
+ }
+ }
+ },
+ "success_all_fields_body": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "get_raw_data_success_all_fields_body",
+ "allOf": [
+ { "$ref": "get_raw_data.json#/success_body" },
+ {
+ "properties": {
+ "fields": {
+ "minItems": 28,
+ "maxItems": 28
+ },
+ "data": {
+ "items": {
+ "minItems": 28,
+ "maxItems": 28
+ }
+ }
+ }
+ }
+ ]
+ },
+ "success_0": {
+ "status_code": 200,
+ "body": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "get_raw_data_success_0_body",
+ "allOf": [
+ { "$ref": "get_raw_data.json#/success_all_fields_body" },
+ {
+ "properties": {
+ "data": {
+ "minItems": 10000,
+ "maxItems": 10000
+ }
+ }
+ }
+ ]
+ }
+ },
+ "success_16500": {
+ "status_code": 200,
+ "body": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "get_raw_data_success_16500_body",
+ "allOf": [
+ { "$ref": "get_raw_data.json#/success_all_fields_body" },
+ {
+ "properties": {
+ "data": {
+ "minItems": 66,
+ "maxItems": 66
+ }
+ }
+ }
+ ]
+ }
+ },
+ "success_fields_and_filters": {
+ "status_code": 200,
+ "body": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "allOf": [
+ { "$ref": "get_raw_data.json#/success_body" },
+ {
+ "properties": {
+ "fields": {
+ "items": {
+ "enum": [ "Nodes", "Wall Time" ]
+ },
+ "minItems": 2,
+ "maxItems": 2
+ },
+ "data": {
+ "items": {
+ "minItems": 2,
+ "maxItems": 2
+ },
+ "minItems": 29,
+ "maxItems": 29
+ }
+ }
+ }
+ ]
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data_limit.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data_limit.json
new file mode 100644
index 0000000000..1a1e6fd8f7
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data_limit.json
@@ -0,0 +1,7 @@
+{
+ "status_code": 200,
+ "body": {
+ "success": true,
+ "data": 10000
+ }
+}
From 07a7c02786b377f51d7dc85fa6741f9881d51afd Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Tue, 9 May 2023 16:41:06 -0400
Subject: [PATCH 11/21] Refactor `WarehouseControllerProviderTest`
to use a constant test group and make use of an inherited method
for running multiple valid token tests on the raw data endpoint.
---
.../Rest/WarehouseControllerProviderTest.php | 56 +++++++++----------
1 file changed, 25 insertions(+), 31 deletions(-)
diff --git a/tests/integration/lib/Rest/WarehouseControllerProviderTest.php b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
index dca10550a2..6c68ab57de 100644
--- a/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
+++ b/tests/integration/lib/Rest/WarehouseControllerProviderTest.php
@@ -7,6 +7,11 @@
class WarehouseControllerProviderTest extends TokenAuthTest
{
+ /**
+ * Directory containing test artifact files.
+ */
+ private static $TEST_GROUP = 'integration/rest/warehouse';
+
private static $helper;
public static function setUpBeforeClass()
@@ -146,46 +151,35 @@ public function testGetRawData($role, $tokenType, $testKey)
parent::runTokenAuthTest(
$role,
$tokenType,
- 'integration/rest/warehouse',
+ self::$TEST_GROUP,
'get_raw_data',
$testKey
);
}
/**
- * dataProvider for testGetRawData.
+ * dataProvider for @see testGetRawData().
*/
public function provideGetRawData()
{
- // Get the test keys out of the test input artifact file.
- $input = parent::loadJsonTestArtifact(
- 'integration/rest/warehouse',
- 'get_raw_data',
- 'input'
+ $testData = TokenAuthTest::provideTokenAuthTestDataWithMultipleKeys(
+ self::$TEST_GROUP,
+ 'get_raw_data'
);
- unset($input['$path']);
- $testKeys = array_keys($input);
-
- // Build the arrays of test data — one for each role, token type, and
- // test key.
- $testData = [];
- foreach (parent::provideTokenAuthTestData() as $roleAndTokenType) {
- list($role, $tokenType) = $roleAndTokenType;
- if ('valid_token' === $tokenType) {
- foreach ($testKeys as $testKey) {
- // Only run the non-default valid token test for one
- // non-public user to make the tests take less time.
- if (
- 'pub' !== $role
- && 'usr' !== $role
- && 'defaults' !== $testKey
- ) {
- continue;
- }
- $testData[] = [$role, $tokenType, $testKey];
- }
- } else {
- $testData[] = [$role, $tokenType, 'defaults'];
+ // Only run the non-default valid token tests for one non-public user
+ // to make the tests take less time overall.
+ $testNames = array_keys($testData);
+ foreach ($testNames as $testName) {
+ if (
+ // If the user is other than 'usr',
+ 1 !== preg_match('/^usr-/', $testName)
+ // and the token type is 'valid_token',
+ && 1 === preg_match('/-valid_token-/', $testName)
+ // and the test key is not 'defaults',
+ && 1 !== preg_match('/-defaults$/', $testName)
+ ) {
+ // Remove the test from the list.
+ unset($testData[$testName]);
}
}
return $testData;
@@ -199,7 +193,7 @@ public function testGetRawDataLimit($role, $tokenType)
parent::runTokenAuthTest(
$role,
$tokenType,
- 'integration/rest/warehouse',
+ self::$TEST_GROUP,
'get_raw_data_limit'
);
}
From 0597c055f52ef5be220d085c9429c55138c1668b Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Tue, 9 May 2023 17:48:59 -0400
Subject: [PATCH 12/21] Refactor and add docblock.
Prefer to include `Exception` in `use` statement rather than
prepending with backslash, and prefer `[]` to `array()`.
---
.../WarehouseControllerProvider.php | 72 +++++++++++++------
1 file changed, 52 insertions(+), 20 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 0a9f2ba601..fa5dd4f589 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -10,6 +10,7 @@
use DataWarehouse\Query\Exceptions\NotFoundException;
use DataWarehouse\Query\Exceptions\BadRequestException;
use DataWarehouse\Query\Model\WhereCondition;
+use Exception;
use Models\Services\Acls;
use Models\Services\Parameters;
use Models\Services\Realms;
@@ -953,7 +954,7 @@ public function getQuickFilters(Request $request, Application $app)
try {
$multipleProvidersSupported = \xd_utilities\getConfiguration('features', 'multiple_service_providers') === 'on';
}
- catch(\Exception $e){
+ catch(Exception $e){
$multipleProvidersSupported = false;
}
@@ -1575,7 +1576,7 @@ private function getJobDataSet(XDUser $user, $realm, $jobId, $action)
* @param string $action the parent action that called this function.
* @param string $actionName the child action that called this function.
* @return \Symfony\Component\HttpFoundation\JsonResponse
- * @throws \Exception
+ * @throws Exception
*/
private function getJobExecutable(Application $app, \XDUser $user, $realm, $jobId, $action, $actionName)
{
@@ -1585,7 +1586,7 @@ private function getJobExecutable(Application $app, \XDUser $user, $realm, $jobI
$execInfo = $query->getJobExecutableInfo($user, $jobId);
if (count($execInfo) === 0) {
- throw new \Exception(
+ throw new Exception(
"Executable information unavailable for $realm $jobId",
500
);
@@ -1629,7 +1630,7 @@ private function atosrecurse(array $values)
* @param string $action
* @return \Symfony\Component\HttpFoundation\JsonResponse
* @throws BadRequestException
- * @throws \Exception
+ * @throws Exception
*/
private function processJobNodeTimeSeriesRequest(
Application $app,
@@ -2191,6 +2192,37 @@ private function getUserStore(\XDUser $user, $realm)
return new \UserStorage($user, $container);
}
+ /**
+ * Get rows of raw data from the data warehouse. Requires API token
+ * authorization.
+ *
+ * The request should contain the following parameters:
+ * - start_date: start of date range for which to get data.
+ * - end_date: end of date range for which to get data.
+ * - realm: data realm for which to get data.
+ *
+ * It can also contain the following optional parameters:
+ * - fields: list of identifiers of fields to get (if not provided, all
+ * fields are gotten).
+ * - filters: mapping of dimension identifiers to their possible values.
+ * Results will only be included whose values for each of the
+ * given dimensions match one of the corresponding given values.
+ * - offset: starting row index of data to get.
+ *
+ * If successful, the response will include the following keys:
+ * - success: true.
+ * - fields: array containing the 'display' property of each field gotten.
+ * - data: array of arrays containing the field values gotten.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws BadRequestException if any of the required parameters are not
+ * included; if an invalid start date, end
+ * date, realm, field identifier, or filter key
+ * is provided; if the end date is before the
+ * start date; or if the offset is negative.
+ */
public function getRawData(Request $request, Application $app)
{
$user = $this->authenticateToken($request);
@@ -2207,7 +2239,7 @@ public function getRawData(Request $request, Application $app)
$limit,
$params['offset']
);
- } catch (\Exception $e) {
+ } catch (Exception $e) {
if (preg_match('/fields/', $e->getMessage())) {
throw new BadRequestException('Invalid fields.');
} else {
@@ -2215,26 +2247,26 @@ public function getRawData(Request $request, Application $app)
}
}
$data = $this->parseRawDataBatchDataset($dataset);
- return $app->json(array(
+ return $app->json([
'success' => true,
'fields' => $dataset->getHeader(),
'data' => $data
- ));
+ ]);
}
public function getRawDataLimit(Request $request, Application $app)
{
$this->authenticateToken($request);
$limit = $this->getConfiguredRawDataLimit();
- return $app->json(array(
+ return $app->json([
'success' => true,
'data' => $limit
- ));
+ ]);
}
private function validateRawDataParams($request, $user)
{
- $params = array();
+ $params = [];
list(
$params['start_date'], $params['end_date']
) = $this->validateRawDataDateParams($request);
@@ -2260,10 +2292,10 @@ private function getRawDataQuery($params)
$realmManager = new RealmManager();
$className = $realmManager->getRawDataQueryClass($params['realm']);
$query = new $className(
- array(
+ [
'start_date' => $params['start_date'],
'end_date' => $params['end_date']
- ),
+ ],
'batch'
);
$query = $this->setRawDataQueryFilters($query, $params);
@@ -2279,11 +2311,11 @@ private function getRawDataLogger()
{
return Log::factory(
'data-warehouse-raw-data-rest',
- array(
+ [
'console' => false,
'file' => false,
'mail' => false
- )
+ ]
);
}
@@ -2297,7 +2329,7 @@ private function getConfiguredRawDataLimit()
private function parseRawDataBatchDataset($dataset)
{
- $data = array();
+ $data = [];
foreach ($dataset as $record) {
$data[] = $record;
}
@@ -2321,7 +2353,7 @@ private function validateRawDataDateParams($request)
'End date cannot be less than start date.'
);
}
- return array($startDate->format('Y-m-d'), $endDate->format('Y-m-d'));
+ return [$startDate->format('Y-m-d'), $endDate->format('Y-m-d')];
}
private function validateRawDataFieldsParam($request)
@@ -2339,7 +2371,7 @@ private function validateRawDataFiltersParams($request, $queryDescripters)
$filters = null;
$filtersParam = $request->get('filters');
if (!is_null($filtersParam)) {
- $filters = array();
+ $filters = [];
foreach ($filtersParam as $filterKey => $filterValuesStr) {
$filters[$filterKey] = $this->validateRawDataFilterParam(
$queryDescripters,
@@ -2355,15 +2387,15 @@ private function setRawDataQueryFilters($query, $params)
{
if (count($params['filters']) > 0) {
$f = new stdClass();
- $f->{'data'} = array();
+ $f->{'data'} = [];
foreach ($params['filters'] as $dimension => $values) {
foreach ($values as $value) {
- $f->{'data'}[] = (object) array(
+ $f->{'data'}[] = (object) [
'id' => "$dimension=$value",
'value_id' => $value,
'dimension_id' => $dimension,
'checked' => 1,
- );
+ ];
}
}
$query->setFilters($f);
From 7f8c5cdb9c49f2fe735dc464d0afb3b1445eea78 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Tue, 9 May 2023 17:55:27 -0400
Subject: [PATCH 13/21] Add docblock.
---
.../WarehouseControllerProvider.php | 23 +++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index fa5dd4f589..83f126d5b6 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2193,8 +2193,8 @@ private function getUserStore(\XDUser $user, $realm)
}
/**
- * Get rows of raw data from the data warehouse. Requires API token
- * authorization.
+ * Endpoint to get rows of raw data from the data warehouse. Requires API
+ * token authorization.
*
* The request should contain the following parameters:
* - start_date: start of date range for which to get data.
@@ -2254,6 +2254,25 @@ public function getRawData(Request $request, Application $app)
]);
}
+ /**
+ * Endpoint to get the maximum number of rows that can be returned in a
+ * single response from the raw data endpoint (@see getRawData()). Requires
+ * API token authorization.
+ *
+ * No parameters.
+ *
+ * If successful, the response will include the following keys:
+ * - success: true.
+ * - data: integer value obtained from the 'rest_raw_row_limit' setting in
+ * the 'datawarehouse' section of the portal_settings.ini
+ * configuration file.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws Exception if there is no setting for 'rest_raw_row_limit' in
+ * the 'datawarehouse' section of portal_settings.ini.
+ */
public function getRawDataLimit(Request $request, Application $app)
{
$this->authenticateToken($request);
From aa825ac2eb1c58fac0bb313642c723fb673900ef Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Tue, 9 May 2023 18:03:34 -0400
Subject: [PATCH 14/21] Add docblock
and use `parent::` to hint that the `authenticateToken()` method
is defined in the parent class.
---
.../WarehouseControllerProvider.php | 21 +++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 83f126d5b6..615e9598cd 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2225,7 +2225,7 @@ private function getUserStore(\XDUser $user, $realm)
*/
public function getRawData(Request $request, Application $app)
{
- $user = $this->authenticateToken($request);
+ $user = parent::authenticateToken($request);
$params = $this->validateRawDataParams($request, $user);
$query = $this->getRawDataQuery($params);
$logger = $this->getRawDataLogger();
@@ -2275,7 +2275,7 @@ public function getRawData(Request $request, Application $app)
*/
public function getRawDataLimit(Request $request, Application $app)
{
- $this->authenticateToken($request);
+ parent::authenticateToken($request);
$limit = $this->getConfiguredRawDataLimit();
return $app->json([
'success' => true,
@@ -2283,6 +2283,15 @@ public function getRawDataLimit(Request $request, Application $app)
]);
}
+ /**
+ * Validate the parameters of the request from the given user to the raw
+ * data endpoint (@see getRawData()).
+ *
+ * @param Request $request
+ * @param XDUser $user
+ * @return array of validated parameter values.
+ * @throws BadRequestException if any of the parameters are invalid.
+ */
private function validateRawDataParams($request, $user)
{
$params = [];
@@ -2306,6 +2315,14 @@ private function validateRawDataParams($request, $user)
return $params;
}
+ /**
+ * Get the corresponding query for a request to get raw data with the given
+ * parameters.
+ *
+ * @param array $params validated parameters
+ * (@see validateRawDataParams()).
+ * @return \DataWarehouse\Query\RawQuery
+ */
private function getRawDataQuery($params)
{
$realmManager = new RealmManager();
From 1bdbac9de81b8a5195b90b44d4159d22084d6b58 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Mon, 15 May 2023 10:40:49 -0400
Subject: [PATCH 15/21] Refactor and add comments for readability.
---
classes/DataWarehouse/Data/BatchDataset.php | 27 +++++++++++++--------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php
index 9753c50834..ff24ce933b 100644
--- a/classes/DataWarehouse/Data/BatchDataset.php
+++ b/classes/DataWarehouse/Data/BatchDataset.php
@@ -87,15 +87,15 @@ class BatchDataset extends Loggable implements Iterator
* @param RawQuery $query
* @param XDUser $user
* @param LoggerInterface $logger
- * @param array $fields
- * @param int $limit
+ * @param array|null $fieldAliases
+ * @param int|null $limit
* @param int $offset
*/
public function __construct(
RawQuery $query,
XDUser $user,
LoggerInterface $logger = null,
- $fields = null,
+ $fieldAliases = null,
$limit = null,
$offset = 0
) {
@@ -114,15 +114,22 @@ public function __construct(
}
$rawStatsConfig = RawStatisticsConfiguration::factory();
- $this->fields = $rawStatsConfig->getBatchExportFieldDefinitions($query->getRealmName());
- if ($fields !== null) {
- foreach ($this->fields as $index => $field) {
- if (!in_array($field['alias'], $fields)) {
- unset($this->fields[$index]);
+ $this->fields = $rawStatsConfig->getBatchExportFieldDefinitions(
+ $query->getRealmName()
+ );
+ // If an array of field aliases has been provided, filter out the
+ // fields whose aliases are not in the list, and make sure all of the
+ // provided field aliases are valid.
+ if (!is_null($fieldAliases)) {
+ $this->fields = array_filter(
+ $this->fields,
+ function ($field) use ($fieldAliases) {
+ return in_array($field['alias'], $fieldAliases);
}
- }
+ );
+ // Renumber the indexes.
$this->fields = array_values($this->fields);
- if (count($fields) !== count($this->fields)) {
+ if (count($fieldAliases) !== count($this->fields)) {
throw new Exception(
get_class($this) . ': invalid fields specified.'
);
From a791f11ebcd05bf5fc25da33f34b5de803244e9f Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Mon, 15 May 2023 16:52:25 -0400
Subject: [PATCH 16/21] Refactor field alias validation
and have it include all the invalid fields in the exception and
response message.
---
classes/DataWarehouse/Data/BatchDataset.php | 25 ++++++++++++-------
.../WarehouseControllerProvider.php | 4 +--
docs/xdmod-rest-schema.json | 2 +-
.../rest/warehouse/input/get_raw_data.json | 2 +-
.../rest/warehouse/output/get_raw_data.json | 2 +-
5 files changed, 21 insertions(+), 14 deletions(-)
diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php
index ff24ce933b..22cdf126e0 100644
--- a/classes/DataWarehouse/Data/BatchDataset.php
+++ b/classes/DataWarehouse/Data/BatchDataset.php
@@ -117,10 +117,22 @@ public function __construct(
$this->fields = $rawStatsConfig->getBatchExportFieldDefinitions(
$query->getRealmName()
);
- // If an array of field aliases has been provided, filter out the
- // fields whose aliases are not in the list, and make sure all of the
- // provided field aliases are valid.
- if (!is_null($fieldAliases)) {
+ // If an array of field aliases has been provided,
+ if (is_array($fieldAliases)) {
+ // Validate the provided field aliases.
+ $validFieldAliases = array_column($this->fields, 'alias');
+ $invalidFieldAliases = array_diff(
+ $fieldAliases,
+ $validFieldAliases
+ );
+ if (count($invalidFieldAliases) > 0) {
+ throw new Exception(
+ "Invalid fields specified: '"
+ . join("', '", $invalidFieldAliases)
+ . "'."
+ );
+ }
+ // Filter out the fields whose aliases were not provided.
$this->fields = array_filter(
$this->fields,
function ($field) use ($fieldAliases) {
@@ -129,11 +141,6 @@ function ($field) use ($fieldAliases) {
);
// Renumber the indexes.
$this->fields = array_values($this->fields);
- if (count($fieldAliases) !== count($this->fields)) {
- throw new Exception(
- get_class($this) . ': invalid fields specified.'
- );
- }
}
$this->limit = $limit;
$this->offset = $offset;
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 615e9598cd..31e035f2f8 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2240,8 +2240,8 @@ public function getRawData(Request $request, Application $app)
$params['offset']
);
} catch (Exception $e) {
- if (preg_match('/fields/', $e->getMessage())) {
- throw new BadRequestException('Invalid fields.');
+ if (preg_match('/Invalid fields specified/', $e->getMessage())) {
+ throw new BadRequestException($e->getMessage());
} else {
throw $e;
}
diff --git a/docs/xdmod-rest-schema.json b/docs/xdmod-rest-schema.json
index 3995deb7ac..56e6a11480 100644
--- a/docs/xdmod-rest-schema.json
+++ b/docs/xdmod-rest-schema.json
@@ -3060,7 +3060,7 @@
"totalCount": 0,
"results": [],
"data": [],
- "message": "Invalid fields.",
+ "message": "Invalid fields specified: 'foo', 'bar'.",
"code": 104
}
},
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json b/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json
index fbea8011f6..f304bae9ff 100644
--- a/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/input/get_raw_data.json
@@ -67,7 +67,7 @@
"start_date": "2017-01-01",
"end_date": "2017-01-01",
"realm": "Jobs",
- "fields": "asdf"
+ "fields": "asdf,jkl;"
}
}
},
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json
index 40ce844fbd..37b5612da3 100644
--- a/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json
+++ b/tests/artifacts/xdmod/integration/rest/warehouse/output/get_raw_data.json
@@ -69,7 +69,7 @@
"body": {
"$ref-with-overwrite": "${INTEGRATION_ROOT}/base/error_body.json",
"$overwrite": {
- "message": "Invalid fields.",
+ "message": "Invalid fields specified: 'asdf', 'jkl;'.",
"code": 104
}
}
From 1969539fc65575007843b8304778d936eb294d2a Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Mon, 15 May 2023 16:56:07 -0400
Subject: [PATCH 17/21] Remove hard-coded SUPReMM null filtering
since this can/should be handled by the client.
---
classes/Rest/Controllers/WarehouseControllerProvider.php | 5 -----
1 file changed, 5 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 31e035f2f8..e929536de9 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2335,11 +2335,6 @@ private function getRawDataQuery($params)
'batch'
);
$query = $this->setRawDataQueryFilters($query, $params);
- if ('SUPREMM' === $params['realm']) {
- $query->addWhereCondition(
- new WhereCondition('jf.cpu_user', 'IS NOT', 'NULL')
- );
- }
return $query;
}
From 786af339ed39d02eb2c303acbedbdb2992f5c86c Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Mon, 15 May 2023 17:51:55 -0400
Subject: [PATCH 18/21] Rename method and add method docblocks.
---
.../WarehouseControllerProvider.php | 88 ++++++++++++++++++-
1 file changed, 86 insertions(+), 2 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index e929536de9..8fce25d8a2 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2303,7 +2303,7 @@ private function validateRawDataParams($request, $user)
if (empty($queryDescripters)) {
throw new BadRequestException('Invalid realm.');
}
- $params['fields'] = $this->validateRawDataFieldsParam($request);
+ $params['fields'] = $this->getRawDataFieldsArray($request);
$params['filters'] = $this->validateRawDataFiltersParams(
$request,
$queryDescripters
@@ -2338,6 +2338,11 @@ private function getRawDataQuery($params)
return $query;
}
+ /**
+ * Generate a database logger for the raw data queries.
+ *
+ * @return \CCR\Logger
+ */
private function getRawDataLogger()
{
return Log::factory(
@@ -2350,6 +2355,16 @@ private function getRawDataLogger()
);
}
+ /**
+ * Get the value configured in the portal settings for the maximum number
+ * of rows that can be returned in a single response from the raw data
+ * endpoint.
+ *
+ * @return int
+ * @throws Exception if the 'datawarehouse' section and/or the
+ * 'rest_raw_row_limit' option have not been set in the
+ * portal configuration.
+ */
private function getConfiguredRawDataLimit()
{
return intval(\xd_utilities\getConfiguration(
@@ -2358,6 +2373,12 @@ private function getConfiguredRawDataLimit()
));
}
+ /**
+ * Parse the given dataset into an array of records.
+ *
+ * @param BatchDataset $dataset
+ * @return array of records obtained by iterating over the dataset.
+ */
private function parseRawDataBatchDataset($dataset)
{
$data = [];
@@ -2367,6 +2388,17 @@ private function parseRawDataBatchDataset($dataset)
return $data;
}
+ /**
+ * Validate the 'start_date' and 'end_date' parameters of the given request
+ * to the raw data endpoint (@see getRawData()).
+ *
+ * @param Request $request
+ * @return array containing the validated start and end dates in Y-m-d
+ * format.
+ * @throws BadRequestException if the start and/or end dates are not
+ * provided or are not valid ISO 8601 dates or
+ * the end date is less than the start date.
+ */
private function validateRawDataDateParams($request)
{
$startDate = $this->getDateFromISO8601Param(
@@ -2387,7 +2419,16 @@ private function validateRawDataDateParams($request)
return [$startDate->format('Y-m-d'), $endDate->format('Y-m-d')];
}
- private function validateRawDataFieldsParam($request)
+ /**
+ * Get the array of field aliases from the given request to the raw data
+ * endpoint (@see getRawData()), e.g., the parameter 'fields=foo,bar,baz'
+ * results in ['foo', 'bar', 'baz'].
+ *
+ * @param Request $request
+ * @return array|null containing the field aliases parsed from the request,
+ * if provided.
+ */
+ private function getRawDataFieldsArray($request)
{
$fields = null;
$fieldsStr = $this->getStringParam($request, 'fields', false);
@@ -2397,6 +2438,21 @@ private function validateRawDataFieldsParam($request)
return $fields;
}
+ /**
+ * Validate the optional 'filters' parameter of the given request to the
+ * raw data endpoint (@see getRawData()), e.g., the parameter
+ * 'filters[foo]=bar,baz' results in ['foo' => ['bar', 'baz']].
+ *
+ * @param Request $request
+ * @param array $queryDescripters the set of dimensions the user is
+ * authorized to see based on their assigned
+ * ACLs.
+ * @return array whose keys are the validated filter keys (they must be
+ * valid dimensions the user is authorized to see) and whose
+ * values are arrays of the provided string values.
+ * @throws BadRequestException if any of the filter keys are invalid
+ * dimension names.
+ */
private function validateRawDataFiltersParams($request, $queryDescripters)
{
$filters = null;
@@ -2414,6 +2470,18 @@ private function validateRawDataFiltersParams($request, $queryDescripters)
return $filters;
}
+ /**
+ * Given a raw data query and a mapping of dimension names to possible
+ * values, set the query to filter out records whose value for the given
+ * dimension does not match any of the provided values.
+ *
+ * @param \DataWarehouse\Query\RawQuery $query
+ * @param array $params containing a 'filters' key whose value is an
+ * associative array of dimensions and dimension
+ * values.
+ * @return \DataWarehouse\Query\RawQuery the query with the filters
+ * applied.
+ */
private function setRawDataQueryFilters($query, $params)
{
if (count($params['filters']) > 0) {
@@ -2434,6 +2502,22 @@ private function setRawDataQueryFilters($query, $params)
return $query;
}
+ /**
+ * Validate a specific filter from the 'filters' parameter of a request to
+ * the raw data endpoint (@see getRawData()), and return the parsed array
+ * of values for that filter (e.g., 'foo,bar,baz' becomes ['foo', 'bar',
+ * 'baz']).
+ *
+ * @param Request $request
+ * @param array $queryDescripters the set of dimensions the user is
+ * authorized to see based on their assigned
+ * ACLs.
+ * @param string $filterKey the label of a dimension.
+ * @param string $filerValuesStr a comma-separated string.
+ * @return array
+ * @throws BadRequestException if the filter key is an invalid dimension
+ * name.
+ */
private function validateRawDataFilterParam(
$queryDescripters,
$filterKey,
From 239012a266d1c77c9bb801e36abd1dbc495fbe27 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Tue, 16 May 2023 09:41:37 -0400
Subject: [PATCH 19/21] Refactor for readability.
---
.../WarehouseControllerProvider.php | 71 +++++++++++++------
1 file changed, 50 insertions(+), 21 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 8fce25d8a2..58547f6a03 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2202,9 +2202,9 @@ private function getUserStore(\XDUser $user, $realm)
* - realm: data realm for which to get data.
*
* It can also contain the following optional parameters:
- * - fields: list of identifiers of fields to get (if not provided, all
+ * - fields: list of aliases of fields to get (if not provided, all
* fields are gotten).
- * - filters: mapping of dimension identifiers to their possible values.
+ * - filters: mapping of dimension names to their possible values.
* Results will only be included whose values for each of the
* given dimensions match one of the corresponding given values.
* - offset: starting row index of data to get.
@@ -2219,7 +2219,7 @@ private function getUserStore(\XDUser $user, $realm)
* @return \Symfony\Component\HttpFoundation\JsonResponse
* @throws BadRequestException if any of the required parameters are not
* included; if an invalid start date, end
- * date, realm, field identifier, or filter key
+ * date, realm, field alias, or filter key
* is provided; if the end date is before the
* start date; or if the offset is negative.
*/
@@ -2230,23 +2230,14 @@ public function getRawData(Request $request, Application $app)
$query = $this->getRawDataQuery($params);
$logger = $this->getRawDataLogger();
$limit = $this->getConfiguredRawDataLimit();
- try {
- $dataset = new BatchDataset(
- $query,
- $user,
- $logger,
- $params['fields'],
- $limit,
- $params['offset']
- );
- } catch (Exception $e) {
- if (preg_match('/Invalid fields specified/', $e->getMessage())) {
- throw new BadRequestException($e->getMessage());
- } else {
- throw $e;
- }
- }
- $data = $this->parseRawDataBatchDataset($dataset);
+ $dataset = $this->getRawBatchDataset(
+ $user,
+ $params,
+ $query,
+ $logger,
+ $limit
+ );
+ $data = $this->parseRawBatchDataset($dataset);
return $app->json([
'success' => true,
'fields' => $dataset->getHeader(),
@@ -2373,13 +2364,51 @@ private function getConfiguredRawDataLimit()
));
}
+ /**
+ * Get a raw batch dataset from the warehouse.
+ *
+ * @param XDUser $user
+ * @param array $params validated parameter values.
+ * @param \DataWarehouse\Query\RawQuery $query
+ * @param \CCR\Logger
+ * @param int $limit maximum number of rows to get.
+ * @return BatchDataset
+ * @throws Exception if the 'fields' parameter contains invalid field
+ * aliases.
+ */
+ private function getRawBatchDataset(
+ $user,
+ $params,
+ $query,
+ $logger,
+ $limit
+ ) {
+ try {
+ $dataset = new BatchDataset(
+ $query,
+ $user,
+ $logger,
+ $params['fields'],
+ $limit,
+ $params['offset']
+ );
+ return $dataset;
+ } catch (Exception $e) {
+ if (preg_match('/Invalid fields specified/', $e->getMessage())) {
+ throw new BadRequestException($e->getMessage());
+ } else {
+ throw $e;
+ }
+ }
+ }
+
/**
* Parse the given dataset into an array of records.
*
* @param BatchDataset $dataset
* @return array of records obtained by iterating over the dataset.
*/
- private function parseRawDataBatchDataset($dataset)
+ private function parseRawBatchDataset($dataset)
{
$data = [];
foreach ($dataset as $record) {
From bf75bce493512318c545e486a0993f3f58bf8a2e Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Thu, 18 May 2023 11:00:54 -0400
Subject: [PATCH 20/21] Fully remove unused querygroup parameter
that became unused in https://github.com/ubccr/xdmod/commit/b72a5875f6218fd229303145c7e60919b5a73a80
---
classes/Rest/Controllers/WarehouseControllerProvider.php | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 58547f6a03..050b34b43c 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -740,10 +740,7 @@ public function getRealms(Request $request, Application $app)
{
$user = $this->authorize($request);
- // Get parameters.
- $this->getStringParam($request, 'querygroup', false, self::_DEFAULT_QUERY_GROUP);
-
- // Get the realms for the query group and the user's active role.
+ // Get the realms for the user's active role.
$realms = Realms::getRealmsForUser($user);
// Return the realms found.
From 490a53d52d296ac7e0bbff720ed7e33a72dc108b Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Thu, 18 May 2023 13:05:26 -0400
Subject: [PATCH 21/21] Fix PHP warning.
---
classes/Rest/Controllers/WarehouseControllerProvider.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 050b34b43c..0385e1c15b 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -2510,7 +2510,7 @@ private function validateRawDataFiltersParams($request, $queryDescripters)
*/
private function setRawDataQueryFilters($query, $params)
{
- if (count($params['filters']) > 0) {
+ if (is_array($params['filters']) && count($params['filters']) > 0) {
$f = new stdClass();
$f->{'data'} = [];
foreach ($params['filters'] as $dimension => $values) {