Merge pull request #1181 from zonemaster/develop

Merge develop into master (Backend)
zonemaster · Jul 1, 2024 · d5fe5db · d5fe5db
2 parents 44e9c07 + 4501fcc
commit d5fe5db
Show file tree

Hide file tree

Showing 13 changed files with 327 additions and 46 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,8 +1,8 @@
-dist: jammy
+dist: focal
 
 language: perl
 
-perl: "5.36"
+perl: "5.38"
 
 stages:
   - test
@@ -13,16 +13,17 @@ script:
 jobs:
   include:
     - stage: test
-      env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
-    - env: TARGET=MySQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_mysql_backend_config.ini
+      env: TARGET=SQLite ZONEMASTER_TRAVIS_TESTING=1 ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+    - env: TARGET=MySQL ZONEMASTER_TRAVIS_TESTING=1 ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_mysql_backend_config.ini
       services: mysql
-    - env: TARGET=PostgreSQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
+    - env: TARGET=PostgreSQL ZONEMASTER_TRAVIS_TESTING=1 ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
+      dist: jammy
       services: postgresql
     # Cover supported Perl versions with SQLite
-    - perl: "5.30"
-      env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+    - perl: "5.34"
+      env: TARGET=SQLite ZONEMASTER_TRAVIS_TESTING=1 ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
     - perl: "5.26"
-      env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+      env: TARGET=SQLite ZONEMASTER_TRAVIS_TESTING=1 ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
 
 addons:
   apt:

diff --git a/Changes b/Changes
@@ -1,6 +1,27 @@
 Release history for Zonemaster component Zonemaster-Backend
 
 
+v11.2.0 2024-07-01 (part of Zonemaster v2024.1 release)
+
+ [Release information]
+ - Translations have not been fully updated in this release. They will
+   be updated in an upcoming release.
+ - Migration of the database is required by this release to retain
+   consistency in results in the database.
+
+ [Features]
+ - Fixes regression that caused Backend’s messages not to be
+   translated (#1166)
+
+ [Fixes]
+ - Avoids double UTF-8 encoding in zmb (#1144)
+ - Makes job_status report proper percentages (#1150)
+ - Updates language in experimental RPC API parameters (#1155)
+ - Makes RPCAPI call Translator’s instance() method, instead of new()
+   (#1167)
+ - Bumps DBD::SQLite to 1.66 for Rocky Linux 8 (#1174)
+
+
 v11.1.1 2024-03-28
 
  [Fixes]

diff --git a/MANIFEST b/MANIFEST
@@ -53,6 +53,7 @@ share/locale/sv/LC_MESSAGES/Zonemaster-Backend.mo
 share/Makefile
 share/patch/patch_db_zonemaster_backend_ver_9.0.0.pl
 share/patch/patch_db_zonemaster_backend_ver_11.1.0.pl
+share/patch/patch_db_zonemaster_backend_ver_11.2.0.pl
 share/patch/patch_mysql_db_zonemaster_backend_ver_1.0.3.pl
 share/patch/patch_mysql_db_zonemaster_backend_ver_5.0.0.pl
 share/patch/patch_mysql_db_zonemaster_backend_ver_5.0.2.pl
@@ -89,3 +90,4 @@ t/test_profile.json
 t/test_validate_syntax.t
 t/TestUtil.pm
 t/validator.t
+t/translator.t
diff --git a/Makefile.PL b/Makefile.PL
@@ -35,19 +35,19 @@ requires
   'Router::Simple::Declare'         => 0,
   'Starman'                         => 0,
   'Try::Tiny'                       => 0.12,
-  'Zonemaster::LDNS'                => 4.000000,  # v4.0.0
-  'Zonemaster::Engine'              => 5.000000,  # v5.0.0
+  'Zonemaster::LDNS'                => 4.000002,  # v4.0.2
+  'Zonemaster::Engine'              => 6.000000,  # v6.0.0
   ;
 
-test_requires 'DBD::SQLite' => 1.4702;
+test_requires 'DBD::SQLite' => 1.66;
 test_requires 'Test::Differences';
 test_requires 'Test::Exception';
 test_requires 'Time::Local' => 1.26;
 test_requires 'Test::NoWarnings';
 
 recommends 'DBD::mysql';
 recommends 'DBD::Pg';
-recommends 'DBD::SQLite';
+recommends 'DBD::SQLite' => 1.66;
 
 install_share;
 

diff --git a/lib/Zonemaster/Backend.pm b/lib/Zonemaster/Backend.pm
@@ -1,6 +1,6 @@
 package Zonemaster::Backend;
 
-our $VERSION = '11.1.1';
+our $VERSION = '11.2.0';
 
 use strict;
 use warnings;

diff --git a/lib/Zonemaster/Backend/DB.pm b/lib/Zonemaster/Backend/DB.pm
@@ -784,7 +784,7 @@ sub process_unfinished_tests {
     my $msg = Zonemaster::Engine::Logger::Entry->new(
         {
             level     => "CRITICAL",
-            module    => "BACKEND_TEST_AGENT",
+            module    => "Backend",
             testcase  => "",
             tag       => "UNABLE_TO_FINISH_TEST",
             args      => { max_execution_time => $test_run_timeout },
@@ -860,7 +860,7 @@ sub process_dead_test {
     my $msg = Zonemaster::Engine::Logger::Entry->new(
         {
             level     => "CRITICAL",
-            module    => "BACKEND_TEST_AGENT",
+            module    => "Backend",
             testcase  => "",
             tag       => "TEST_DIED",
             args      => {},

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -338,10 +338,16 @@ sub test_progress {
 }
 
 # Experimental
-$json_schemas{job_status} = $json_schemas{test_progress};
+$json_schemas{job_status} = joi->object->strict->props(
+    job_id => $zm_validator->test_id->required
+);
 sub job_status {
+    my ( $self, $params ) = @_;
+
+    $params->{test_id} = delete $params->{job_id};
+
     my $result = {
-        progress => test_progress( @_ )
+        progress => $self->test_progress( $params )
     };
     return $result;
 }
@@ -366,9 +372,15 @@ sub get_test_params {
 }
 
 # Experimental
-$json_schemas{job_params} = $json_schemas{get_test_params};
+$json_schemas{job_params} = joi->object->strict->props(
+    job_id => $zm_validator->test_id->required
+);
 sub job_params {
-    return get_test_params( @_ );
+    my ( $self, $params ) = @_;
+
+    $params->{test_id} = delete $params->{job_id};
+
+    return $self->get_test_params( $params );
 }
 
 $json_schemas{get_test_results} = {
@@ -389,7 +401,7 @@ sub get_test_results {
         my $locale = $self->_get_locale( $params );
 
         my $translator;
-        $translator = Zonemaster::Backend::Translator->new;
+        $translator = Zonemaster::Backend::Translator->instance();
 
         my $previous_locale = $translator->locale;
         if ( !$translator->locale( $locale ) ) {
@@ -467,9 +479,29 @@ sub get_test_results {
 }
 
 # Experimental
-$json_schemas{job_results} = $json_schemas{get_test_results};
+$json_schemas{job_results} = {
+    type => 'object',
+    additionalProperties => 0,
+    required => [ 'job_id', 'language' ],
+    properties => {
+        job_id => $zm_validator->test_id->required->compile,
+        language => $zm_validator->language_tag,
+    }
+};
 sub job_results {
-    return get_test_results( @_ );
+    my ( $self, $params ) = @_;
+
+    $params->{id} = delete $params->{job_id};
+
+    my $result = $self->get_test_results( $params );
+
+    return {
+        created_at            => $result->{created_at},
+        job_id                => $result->{hash_id},
+        results               => $result->{results},
+        params                => $result->{params},
+        testcase_descriptions => $result->{testcase_descriptionsd},
+    };
 }
 
 $json_schemas{get_test_history} = {
@@ -513,12 +545,43 @@ sub get_test_history {
 }
 
 # Experimental
-$json_schemas{domain_history} = $json_schemas{get_test_history};
+$json_schemas{domain_history} = {
+    type => 'object',
+    additionalProperties => 0,
+    required => [ 'params' ],
+    properties => {
+        offset => joi->integer->min(0)->compile,
+        limit => joi->integer->min(0)->compile,
+        filter => joi->string->regex('^(?:all|delegated|undelegated)$')->compile,
+        params => {
+            type => 'object',
+            additionalProperties => 0,
+            required => [ 'domain' ],
+            properties => {
+                domain => $zm_validator->domain_name
+            }
+        }
+    }
+};
 sub domain_history {
-    my $result = {
-        history => get_test_history( @_ )
+    my ( $self, $params ) = @_;
+
+    $params->{frontend_params} = delete $params->{params};
+
+    my $results = $self->get_test_history( $params );
+
+    return {
+        history => [
+            map {
+                {
+                    job_id         => $_->{id},
+                    created_at     => $_->{created_at},
+                    overall_result => $_->{overall_result},
+                    undelegated    => $_->{undelegated},
+                }
+            } @$results
+        ],
     };
-    return $result;
 }
 
 $json_schemas{add_api_user} = joi->object->strict->props(
@@ -625,11 +688,52 @@ sub add_batch_job {
 }
 
 # Experimental
-$json_schemas{batch_create} = $json_schemas{add_batch_job};
+$json_schemas{batch_create} = {
+    type => 'object',
+    additionalProperties => 0,
+    required => [ 'username', 'api_key', 'domains' ],
+    properties => {
+        username => $zm_validator->username->required->compile,
+        api_key => $zm_validator->api_key->required->compile,
+        domains => {
+            type => "array",
+            additionalItems => 0,
+            items => $zm_validator->domain_name,
+            minItems => 1
+        },
+        job_params => {
+            type => 'object',
+            additionalProperties => 0,
+            properties => {
+                ipv4 => joi->boolean->compile,
+                ipv6 => joi->boolean->compile,
+                nameservers => {
+                    type => 'array',
+                    items => $zm_validator->nameserver
+                },
+                ds_info => {
+                    type => 'array',
+                    items => $zm_validator->ds_info
+                },
+                profile => $zm_validator->profile_name,
+                client_id => $zm_validator->client_id->compile,
+                client_version => $zm_validator->client_version->compile,
+                config => joi->string->compile,
+                priority => $zm_validator->priority->compile,
+                queue => $zm_validator->queue->compile,
+            }
+        }
+    }
+};
 sub batch_create {
+    my ( $self, $params ) = @_;
+
+    $params->{test_params} = delete $params->{job_params};
+
     my $result = {
-        batch_id => add_batch_job( @_ )
+        batch_id => $self->add_batch_job( $params )
     };
+
     return $result;
 }
 
@@ -655,7 +759,12 @@ sub get_batch_job_result {
 # Experimental
 $json_schemas{batch_status} = $json_schemas{get_batch_job_result};
 sub batch_status {
-    return get_batch_job_result( @_ );
+    my $result = get_batch_job_result( @_ );
+    return {
+        running_count    => $result->{nb_running},
+        finished_count   => $result->{nb_finished},
+        finished_job_ids => $result->{finished_job_ids},
+    };
 }
 
 sub _get_locale {

diff --git a/lib/Zonemaster/Backend/TestAgent.pm b/lib/Zonemaster/Backend/TestAgent.pm
@@ -117,8 +117,7 @@ sub run {
 
                 if ( $entry->{tag} and $entry->{tag} eq 'TEST_CASE_END' ) {
                     $nbr_testcases_finished++;
-                    # limit to max 99%, 100% is reached when data is stored in database
-                    my $progress_percent = int( 99 * $nbr_testcases_finished /  $nbr_testcases_planned );
+                    my $progress_percent = int( 100 * $nbr_testcases_finished /  $nbr_testcases_planned );
                     $self->{_db}->test_progress( $test_id, $progress_percent );
                 }
             }

diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
@@ -30,21 +30,25 @@ Readonly my %TAG_DESCRIPTIONS => (
 );
 
 sub _build_all_tag_descriptions {
+    my ( $class ) = @_;
+
     my $all_tag_descriptions = Zonemaster::Engine::Translator::_build_all_tag_descriptions();
-    $all_tag_descriptions->{BACKEND_TEST_AGENT} = \%TAG_DESCRIPTIONS;
+    $all_tag_descriptions->{Backend} = \%TAG_DESCRIPTIONS;
     return $all_tag_descriptions;
 }
 
 sub translate_tag {
     my ( $self, $hashref ) = @_;
 
     my $entry = Zonemaster::Engine::Logger::Entry->new( { %{ $hashref } } );
-    my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $entry );
 
-    return decode_utf8( $octets );
+    return decode_utf8( $self->SUPER::translate_tag( $entry ) );
 }
 
 sub test_case_description {
-    return decode_utf8(Zonemaster::Engine::Translator::test_case_description(@_));
+    my ( $self, $test_name ) = @_;
+
+    return decode_utf8( $self->SUPER::test_case_description( $test_name ) );
 }
+
 1;
diff --git a/script/zmb b/script/zmb
@@ -3,12 +3,11 @@ use strict;
 use warnings;
 use feature 'say';
 
+use Encode qw( decode_utf8 FB_CROAK );
 use Getopt::Long qw( GetOptionsFromArray :config require_order );
-
-use Pod::Usage;
-
-use JSON::PP;
+use JSON::PP qw( encode_json );
 use LWP::UserAgent;
+use Pod::Usage;
 
 =head1 NAME
 
@@ -33,6 +32,8 @@ This interface is unstable and will change in a future release.
 sub main {
     my @argv = @_;
 
+    @argv = map { decode_utf8( $_, FB_CROAK ) } @argv;
+
     my $opt_help;
     my $opt_verbose;
     my $opt_server = 'http://localhost:5000/';