From d2f7ba34fc06ae9732d15855f94f03cde02deea4 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 15 Sep 2022 13:56:03 +0200 Subject: [PATCH] Charts version can be read and chosen on creation (#155) With this PR the user can now read all charts version via new endpoints. These charts are now fetched in catalog wrapper as well as the last one. The packageVersion in creation request is now passed to helm. Also creating a service which already exists upgrade it instead of failing. This allows user to upgrade there service version. Plus service exposes more helm data. --- .../service/HelmInstallService.java | 11 +++- .../api/controller/pub/CatalogController.java | 60 ++++++++++++++++- .../api/dao/universe/CatalogLoader.java | 17 +++-- .../onyxia/api/services/CatalogService.java | 8 +++ .../api/services/impl/CatalogServiceImpl.java | 18 +++++ .../api/services/impl/HelmAppsService.java | 22 ++----- .../onyxia/model/catalog/CatalogWrapper.java | 19 +++++- .../onyxia/model/dto/UpdateServiceDTO.java | 66 ------------------- .../insee/onyxia/model/helm/Repository.java | 1 + .../insee/onyxia/model/service/Service.java | 51 ++++++++++---- 10 files changed, 169 insertions(+), 104 deletions(-) delete mode 100644 onyxia-model/src/main/java/fr/insee/onyxia/model/dto/UpdateServiceDTO.java diff --git a/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java b/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java index 74eedb27..ca2e799a 100644 --- a/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java +++ b/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java @@ -7,6 +7,8 @@ import io.github.inseefrlab.helmwrapper.model.HelmInstaller; import io.github.inseefrlab.helmwrapper.model.HelmLs; import io.github.inseefrlab.helmwrapper.utils.Command; + +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zeroturnaround.exec.InvalidExitValueException; @@ -28,10 +30,10 @@ public class HelmInstallService { public HelmInstallService() { } - public HelmInstaller installChart(HelmConfiguration configuration,String chart, String namespace, String name, boolean dryRun, File values, - Map env) + public HelmInstaller installChart(HelmConfiguration configuration,String chart, String namespace, String name, String version, + boolean dryRun, File values, Map env) throws InvalidExitValueException, IOException, InterruptedException, TimeoutException { - String command = "helm install "; + String command = "helm upgrade --install "; if (name != null) { command = command.concat(name+ " "); } @@ -40,6 +42,9 @@ public HelmInstaller installChart(HelmConfiguration configuration,String chart, } command = command.concat(chart+" "); command = command.concat("-n "+namespace); + if (StringUtils.isNotBlank(version)) { + command = command.concat(" --version " + version); + } if (values != null) { command = command.concat(" -f " + values.getAbsolutePath()); } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java index 9ebc5363..58375db6 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java @@ -7,6 +7,7 @@ import fr.insee.onyxia.model.catalog.Config.Property; import fr.insee.onyxia.model.catalog.Config.Property.XForm; import fr.insee.onyxia.model.catalog.Config.Property.XOnyxia; +import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.region.Region; import fr.insee.onyxia.model.service.Service; @@ -21,11 +22,12 @@ import org.springframework.web.bind.annotation.RestController; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; @Tag(name = "Public") -@RequestMapping(value={"/api/public/catalog", "/public/catalog"}) +@RequestMapping(value={"/api/public/catalog", "/public/catalog", "/api/public/catalogs", "/public/catalogs"}) @RestController public class CatalogController { @@ -92,6 +94,62 @@ public Pkg getPackage(@PathVariable String catalogId, @PathVariable String packa return pkg; } + @Operation( + summary = "Get a helm chart from a specific catalog by version.", + description = "Get a helm chart from a specific catalog by version, with detailed information on the package including: descriptions, sources, and configuration options.", + parameters = { + @Parameter( + required = true, + name = "catalogId", + description = "Unique ID of the enabled catalog for this Onyxia API.", + in = ParameterIn.PATH + ), + @Parameter( + required = true, + name = "chartName", + description = "Unique name of the chart from the selected catalog.", + in = ParameterIn.PATH + ), + @Parameter( + required = true, + name = "version", + description = "Version of the chart", + in = ParameterIn.PATH + ) + } + ) + @GetMapping("{catalogId}/charts/{chartName}/versions/{version}") + public Chart getChartByVersion(@PathVariable String catalogId, @PathVariable String chartName, @PathVariable String version) { + Chart chart = catalogService.getChartByVersion(catalogId, chartName, version).orElseThrow(NotFoundException::new); + addCustomOnyxiaProperties(chart); + return chart; + } + + @Operation( + summary = "Get all versions of a chart from a specific catalog.", + description = "Get all versions of a chart from a specific catalog, with detailed information on the package including: descriptions, sources, and configuration options.", + parameters = { + @Parameter( + required = true, + name = "catalogId", + description = "Unique ID of the enabled catalog for this Onyxia API.", + in = ParameterIn.PATH + ), + @Parameter( + required = true, + name = "chartName", + description = "Unique name of the chart from the selected catalog.", + in = ParameterIn.PATH + ) + } + ) + @GetMapping("{catalogId}/charts/{chartName}") + public List getCharts(@PathVariable String catalogId, @PathVariable String chartName) { + List charts = catalogService.getCharts(catalogId, chartName).orElseThrow(NotFoundException::new); + charts.stream().forEach(this::addCustomOnyxiaProperties); + return charts; + } + private boolean isCatalogEnabled(Region region, CatalogWrapper catalog) { if (region == null) { return true; diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 1d0cf8f0..4f0ce042 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -24,6 +24,7 @@ import java.io.InputStreamReader; import java.io.Reader; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; @Service public class CatalogLoader { @@ -60,13 +61,17 @@ private void updateHelmRepository(CatalogWrapper cw) { Reader reader = new InputStreamReader(resourceLoader.getResource(cw.getLocation()+"/index.yaml").getInputStream(), "UTF-8"); Repository repository = mapperHelm.readValue(reader, Repository.class); - repository.getPackages().parallelStream().forEach(pkg -> { - try { - refreshPackage(cw, pkg); - } catch (IOException e) { - e.printStackTrace(); - } + repository.getEntries().values().parallelStream().forEach(entry -> { + entry.parallelStream().forEach(pkg -> { + try { + refreshPackage(cw, pkg); + } catch (IOException e) { + e.printStackTrace(); + } + }); }); + repository.setPackages(repository.getEntries().values().stream().map(charts -> charts.get(0)) + .filter(chart -> "application".equalsIgnoreCase(chart.getType())).collect(Collectors.toList())); cw.setCatalog(repository); cw.setLastUpdateTime(System.currentTimeMillis()); } catch (Exception e) { diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/CatalogService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/CatalogService.java index 7f2ebcab..9e8d8b12 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/CatalogService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/CatalogService.java @@ -1,8 +1,12 @@ package fr.insee.onyxia.api.services; +import java.util.List; +import java.util.Optional; + import fr.insee.onyxia.api.configuration.CatalogWrapper; import fr.insee.onyxia.api.configuration.Catalogs; import fr.insee.onyxia.model.catalog.Pkg; +import fr.insee.onyxia.model.helm.Chart; public interface CatalogService { @@ -11,4 +15,8 @@ public interface CatalogService { public CatalogWrapper getCatalogById(String catalogId); public Pkg getPackage(String catalogId, String packageName); + + public Optional getChartByVersion(String catalogId, String chartName, String version); + + public Optional> getCharts(String catalogId, String chartName); } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/CatalogServiceImpl.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/CatalogServiceImpl.java index 9a42e046..df656ac1 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/CatalogServiceImpl.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/CatalogServiceImpl.java @@ -1,5 +1,8 @@ package fr.insee.onyxia.api.services.impl; +import java.util.List; +import java.util.Optional; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -7,6 +10,7 @@ import fr.insee.onyxia.api.configuration.Catalogs; import fr.insee.onyxia.api.services.CatalogService; import fr.insee.onyxia.model.catalog.Pkg; +import fr.insee.onyxia.model.helm.Chart; @Service public class CatalogServiceImpl implements CatalogService { @@ -29,4 +33,18 @@ public Pkg getPackage(String catalogId, String packageName) { return catalogs.getCatalogById(catalogId).getCatalog().getPackageByName(packageName); } + @Override + public Optional> getCharts(String catalogId, String chartName) { + return Optional.ofNullable(catalogs.getCatalogById(catalogId).getCatalog().getEntries().get(chartName)); + } + + @Override + public Optional getChartByVersion(String catalogId, String chartName, String version) { + List charts = catalogs.getCatalogById(catalogId).getCatalog().getEntries().get(chartName); + if (charts != null) { + return charts.stream().filter(c -> c.getVersion().equalsIgnoreCase(version)).findAny(); + } + else return Optional.empty(); + } + } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index 40b221a4..12f6cddb 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -128,7 +128,7 @@ public String getInitScript(String scopeName, XGeneratedContext.Scope scope, Pro File values = File.createTempFile("values", ".yaml"); mapperHelm.writeValue(values, fusion); String namespaceId = kubernetesService.determineNamespaceAndCreateIfNeeded(region, project, user); - HelmInstaller res = getHelmInstallService().installChart(getHelmConfiguration(region, user), catalogId + "/" + pkg.getName(), namespaceId, requestDTO.getName(), requestDTO.isDryRun(), + HelmInstaller res = getHelmInstallService().installChart(getHelmConfiguration(region, user), catalogId + "/" + pkg.getName(), namespaceId, requestDTO.getName(), requestDTO.getPackageVersion(), requestDTO.isDryRun(), values, null); values.delete(); return List.of(res.getManifest()); @@ -201,7 +201,6 @@ public UninstallService destroyService(Region region, Project project, User user private Service getHelmApp(Region region, User user, HelmLs release) { String manifest = getHelmInstallService().getManifest(getHelmConfiguration(region, user), release.getName(), release.getNamespace()); Service service = getServiceFromRelease(region, release, manifest, user); - service.setStatus(findAppStatus(release)); try { service.setStartedAt(helmDateFormat.parse(release.getUpdated()).getTime()); } catch (ParseException e) { @@ -211,6 +210,12 @@ private Service getHelmApp(Region region, User user, HelmLs release) { service.setName(release.getName()); service.setSubtitle(release.getChart()); service.setType(Service.ServiceType.KUBERNETES); + service.setName(release.getName()); + service.setNamespace(release.getNamespace()); + service.setRevision(release.getRevision()); + service.setStatus(release.getStatus()); + service.setUpdated(release.getUpdated()); + service.setAppVersion(release.getAppVersion()); try { String values = getHelmInstallService().getValues(getHelmConfiguration(region, user), release.getName(), release.getNamespace()); JsonNode node = new ObjectMapper().readTree(values); @@ -301,17 +306,4 @@ private Service getServiceFromRelease(Region region, HelmLs release, String mani return service; } - - - private Service.ServiceStatus findAppStatus(HelmLs release) { - if (release.getStatus().equals("deployed")) { - return Service.ServiceStatus.RUNNING; - } else if (release.getStatus().equals("pending")) { - return Service.ServiceStatus.DEPLOYING; - } else { - return Service.ServiceStatus.STOPPED; - } - } - - } diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/CatalogWrapper.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/CatalogWrapper.java index edb21b83..56e9c741 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/CatalogWrapper.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/CatalogWrapper.java @@ -1,11 +1,13 @@ package fr.insee.onyxia.model.catalog; import java.util.List; +import java.util.Map; +import fr.insee.onyxia.model.helm.Chart; public abstract class CatalogWrapper { private List packages; - + private Map> entries; /** * @return the packages */ @@ -28,4 +30,19 @@ public Pkg getPackageByName(String name) { } return null; } + + /** + * @return the packages + */ + public Map> getEntries() { + return entries; + } + + /** + * @param entries the packages to set + */ + public void setEntries(Map> entries) { + this.entries = entries; + } + } \ No newline at end of file diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/dto/UpdateServiceDTO.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/dto/UpdateServiceDTO.java deleted file mode 100644 index d6c472be..00000000 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/dto/UpdateServiceDTO.java +++ /dev/null @@ -1,66 +0,0 @@ -package fr.insee.onyxia.model.dto; - -import java.util.Map; - -import javax.xml.bind.annotation.XmlRootElement; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -@XmlRootElement -@JsonIgnoreProperties(ignoreUnknown = true) -public class UpdateServiceDTO { - int instances = 1; - String serviceId; - Double cpus, mems; - String friendlyName; - Map env; - - public Map getEnv() { - return env; - } - - public void setEnv(Map env) { - this.env = env; - } - - public String getFriendlyName() { - return friendlyName; - } - - public void setFriendlyName(String friendlyName) { - this.friendlyName = friendlyName; - } - - public Double getCpus() { - return cpus; - } - - public void setCpus(Double cpus) { - this.cpus = cpus; - } - - public Double getMems() { - return mems; - } - - public void setMems(Double mems) { - this.mems = mems; - } - - public String getServiceId() { - return serviceId; - } - - public void setServiceId(String serviceId) { - this.serviceId = serviceId; - } - - public int getInstances() { - return instances; - } - - public void setInstances(int instances) { - this.instances = instances; - } - -} diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/helm/Repository.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/helm/Repository.java index 919f3f93..83f1f8e9 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/helm/Repository.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/helm/Repository.java @@ -53,6 +53,7 @@ public void setAdditionalProperty(String name, Object value) { @JsonProperty("entries") public void setEntries(Map> entries) { + super.setEntries(entries); setPackages(entries.values().stream().map(charts -> charts.get(0)).filter(chart -> "application".equalsIgnoreCase(chart.getType())).collect(Collectors.toList())); } } diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java index 3ae68f81..aac82a71 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java @@ -11,17 +11,20 @@ public class Service { private int instances; private double cpus; private double mem; - private ServiceStatus status = ServiceStatus.RUNNING; + private String status; private ServiceType type; private List urls; private List internalUrls; - private String logo; private Map env = new HashMap<>(); private List tasks = new ArrayList<>(); private List events = new ArrayList<>(); private String subtitle; private Monitoring monitoring; private String postInstallInstructions; + private String namespace; + private String revision; + private String updated; + private String appVersion; private long startedAt; @@ -83,11 +86,11 @@ public void setStartedAt(long startedAt) { this.startedAt = startedAt; } - public ServiceStatus getStatus() { + public String getStatus() { return status; } - public void setStatus(ServiceStatus status) { + public void setStatus(String status) { this.status = status; } @@ -107,14 +110,6 @@ public void setUrls(List urls) { this.urls = urls; } - public String getLogo() { - return logo; - } - - public void setLogo(String logo) { - this.logo = logo; - } - public Map getEnv() { return env; } @@ -163,6 +158,38 @@ public void setMonitoring(Monitoring monitoring) { this.monitoring = monitoring; } + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getRevision() { + return revision; + } + + public void setRevision(String revision) { + this.revision = revision; + } + + public String getUpdated() { + return updated; + } + + public void setUpdated(String updated) { + this.updated = updated; + } + + public String getAppVersion() { + return appVersion; + } + + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + public void setPostInstallInstructions(String postInstallInstructions) { this.postInstallInstructions = postInstallInstructions; }