Provide override option when import generic assay data (for review)

.circleci/config.yml

@@ -33,7 +33,7 @@ jobs:
             - run:
                 name: "Install yarn at specific version"
                 command:
-                    sudo npm install --global yarn@1.13.0
+                    sudo npm install --global yarn@1.21.1
             - run:
                 name: "Show yarn and node versions"
                 command: |
@@ -106,9 +106,10 @@ jobs:
                     cd $TEST_HOME/local/runtime-config && \
                     ./setup_environment.sh && ./setup_environment.sh >> $BASH_ENV
             - run:
-                name: Generate checksum of data that populates the test database
+                name: Build image and generate checksum of data that populates the test database
                 command: |         
                     $TEST_HOME/local/runtime-config/db_content_fingerprint.sh > /tmp/db_data_md5key
+                no_output_timeout: 1h
             - restore_cache:
                 keys:
                     - v4-e2e-database-files-{{ checksum "/tmp/db_data_md5key" }}

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,7 @@ Describe changes proposed in this pull request:
 
 # Any screenshots or GIFs?
 If this is a new visual feature please add a before/after screenshot or gif
-here with e.g. [GifGrabber](http://www.gifgrabber.com/).
+here with e.g. [Giphy CAPTURE](https://giphy.com/apps/giphycapture) or [Peek](https://github.com/phw/peek)
 
 # Notify reviewers
 Read our [Pull request merging

.github/release-drafter.yml

@@ -4,46 +4,67 @@ categories:
   - title: '🧬 Features'
     labels:
       - 'feature'
+      - 'cl-feature'
+  - title: '✨ Enhancements'
+    labels:
       - 'enhancement'
+      - 'cl-enhancement'
   - title: '🐛 Bug Fixes'
     labels:
       - 'fix'
       - 'bugfix'
       - 'bug'
+      - 'cl-bug'
   - title: '🏎 Performance Tweaks'
     labels:
       - 'performance'
+      - 'cl-performance'
   - title: '🎨 Style Tweaks'
     labels:
       - 'style tweak'
+      - 'cl-style-tweak'
   - title: '⚙️ REST API Changes'
     labels:
       - 'api'
   - title: '⚙️ Changes to handle external APIs'
     labels:
       - 'external api'
+      - 'cl-external-api'
   - title: '🗄 Database Migration'
     labels:
       - 'includes db changes'
       - 'migration'
+      - 'cl-db-migration'
   - title: '⚠️ Deprecation'
     labels:
       - 'deprecation'
+      - 'cl-deprecation'
   - title: '📘 Documentation'
     labels:
       - 'documentation'
-  - title: '🧹 Cleanup'
+      - 'cl-documentation'
+  - title: '🧹 Cleanup & Refactoring 🔧'
     labels:
       - 'cleanup'
+      - 'refactoring'
+      - 'cl-refactoring'
   - title: '👷‍♀️ Testing, Configuration & Deployment'
     labels:
       - 'devops'
+      - 'test'
+      - 'cl-test'
+      - 'cl-devops'
   - title: '🧰 Maintenance'
     labels:
       - 'chore'
       - 'dependencies'
+      - 'cl-chore'
+  - title: '📐 Prototype (Internal Use)'
+    labels:
+      - 'cl-prototype'
 exclude-labels:
   - 'skip-changelog'
+  - 'cl-skip-changelog'
 change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
 template: |
   ## Changes
@@ -53,3 +74,6 @@ template: |
 
   - Backend: https://github.com/cBioPortal/cbioportal/compare/$PREVIOUS_TAG...v$NEXT_PATCH_VERSION
   - Frontend: https://github.com/cBioPortal/cbioportal-frontend/compare/$PREVIOUS_TAG...v$NEXT_PATCH_VERSION
+
+  ## 🏷Notes on versioning and release procedure
+  https://docs.cbioportal.org/4.-development/release-procedure#a-note-on-versioning

.github/stale.yml

@@ -0,0 +1,21 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 90
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 14
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+  - critical
+  - priority
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 1

.github/workflows/dockerimage.yml

@@ -0,0 +1,59 @@
+name: Docker Image CI
+on:
+  push:
+    branches: 
+      - master
+      - release-*
+      - rc
+    tags: '*'
+
+jobs:
+  build_and_publish_web_and_data:
+    if: github.repository == 'cBioPortal/cbioportal'
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout git repo'
+        uses: actions/checkout@v1
+      - name: Extract branch or tag name
+        # The GITHUB_REF variable is like "refs/head/branch_name" or
+        # "refs/tag/tag_name". If the tag is prefixed with v, this is a new
+        # version and we want to push it with the tag "latest" as well because
+        # that is the version we refer to in the docucmentation. One can give 
+        # the same image multiple tags by using ","
+        run: echo "##[set-output name=image_tag_names;]$(echo ${GITHUB_REF##*/} | sed 's/^v/latest,/g')"
+        id: extract_tags
+      - name: 'Docker build with cache'
+        uses: whoan/docker-build-with-cache-action@v4
+        with:
+          username: "${{ secrets.DOCKER_USERNAME }}"
+          password: "${{ secrets.DOCKER_PASSWORD }}"
+          image_name: cbioportal/cbioportal
+          image_tag: ${{ steps.extract_tags.outputs.image_tag_names }}
+          context: .
+          dockerfile: docker/web-and-data/Dockerfile
+          pull_image_and_stages: false
+
+  build_and_publish_web:
+      if: github.repository == 'cBioPortal/cbioportal'
+      runs-on: ubuntu-latest
+      steps:
+        - name: 'Checkout git repo'
+          uses: actions/checkout@v1
+        - name: Extract branch or tag name
+          # For the web docker image we don't publish it as latest
+          # just extract branch/tag name and strip v prefix
+          run: echo "##[set-output name=image_tag_names;]$(echo ${GITHUB_REF##*/} | sed 's/^v//g')"
+          id: extract_tags
+        - name: 'Docker build with cache'
+          uses: whoan/docker-build-with-cache-action@v4
+          with:
+            username: "${{ secrets.DOCKER_USERNAME }}"
+            password: "${{ secrets.DOCKER_PASSWORD }}"
+            image_name: cbioportal/cbioportal
+            image_tag: ${{ steps.extract_tags.outputs.image_tag_names }}-web-shenandoah
+            context: .
+            dockerfile: docker/web/Dockerfile
+            pull_image_and_stages: false
+
+
+# Reference: https://github.com/marketplace/actions/build-docker-images-using-cache

CONTRIBUTING.md

@@ -1,41 +1,49 @@
 # How to contribute
 
-Thank you for contributing to cBioPortal!  This document provides a brief set of guidelines for contributing.
+Thank you for your interest in contributing to cBioPortal! This document provides a brief set of guidelines for contributing.
 
-# Background
+# Who are you?
 
-The cBioPortal currently uses a "fork and pull" model for collaborative software development.
+We are curious to learn more about you! We would love to help you get started! The contributors in our community all have different backgrounds. For instance some people have:
 
-From the [GitHub Help Page of Using Pull Requests](https://help.github.com/articles/using-pull-requests/):
+1. Engineering experience but no to little knowledge of cancer genomics
+2. Knowledge about cancer genomics but no to little engineering experience
+3. No engineering nor cancer genomics experience but an eagerness to contribute
 
-"The fork & pull model lets anyone fork an existing repository and push changes to their personal fork without requiring access be granted to the source repository. The changes must then be pulled into the source repository by the project maintainer. This model reduces the amount of friction for new contributors and is popular with open source projects because it allows people to work independently without upfront coordination."
+if you feel like you don't fall into any of these categories, please reach out so you can help us update the above list 🙂! Note that there are many contributions that can be made to an open source commmunity without coding a single line of code. You can reach us through our [public slack channel](https://slack.cbioportal.org).
 
-## Branches within cBioPortal
+# Join the Slack!
+
+Come and chat with us at https://slack.cbioportal.org 👋
 
-The cBioPortal currently maintains three branches:
+# Making a code contribution
 
- * **master**:  this reflects what is currently running in production on cbioportal.org. Bug fixes and documentation fixes go here.
- * **rc**:  release candidate branch, incorporating all the latest features. You could see our **rc** branch as a development branch where we only accept high quality contributions. Once ready for testing on cbioportal.org/beta a new branch is formed with the name **release-x.y.z**.
- *  **release-x.y.z**: before each release a new branch is created from **rc** that has a name like **release-x.y.z** .This one is usually deployed to www.cbioportal.org/beta. 
+The cBioPortal currently uses a "fork and pull" model for collaborative software development.
+
+From the [GitHub Help Page of Using Pull Requests](https://help.github.com/articles/using-pull-requests/):
+
+"The fork & pull model lets anyone fork an existing repository and push changes to their personal fork without requiring access be granted to the source repository. The changes must then be pulled into the source repository by the project maintainer. This model reduces the amount of friction for new contributors and is popular with open source projects because it allows people to work independently without upfront coordination."
 
 ## Getting Started
 
  * Make sure you have a [GitHub account](https://github.com/signup/free).
  * Create an issue in our issues tracker, assuming one does not already exist.
  * Fork the cbioportal project on GitHub.  For general instructions on forking a GitHub project, see [Forking a Repo](https://help.github.com/articles/fork-a-repo/) and [Syncing a fork](https://help.github.com/articles/syncing-a-fork/).
- * Familiarize yourself with the [project documentation](https://docs.cbioportal.org), including [backend code organization](Backend-Code-Organization.md) and [backend development guidelines](Backend-Development-Guidelines.md).
+ * Familiarize yourself with the [project documentation](https://docs.cbioportal.org), including the [Architecture docs](https://docs.cbioportal.org/2.1-deployment/architecture-overview), the [backend code organization](docs/Backend-Code-Organization.md) and [backend development guidelines](docs/Backend-Development-Guidelines.md).
 
 ## Contributing Code Changes via a Pull Request
 
 Once you have forked the repo, you need to create your code contributions within a new branch of your forked repo.  For general background on creating and managing branches within GitHub, see:  [Git Branching and Merging](https://git-scm.com/book/en/v2/Git-Branching-Basic-Branching-and-Merging).
 
 * To begin, create a topic branch from where you want to base your work.
- * For a new feature, this is usually the **rc branch**.  For documentation and bug fixes, this is usually the **master branch**.
+ * For any change that requires database migrations, this will be the **rc branch**. For all other changes, this will be the **master branch**. For additional details, see [Branches within cBioPortal](#branches-within-cbioportal) below.
 
 You usually create a branch like so:
 
-```git checkout master```
-```git checkout -b [name_of_your_new_branch]```
+```
+git checkout master
+git checkout -b [name_of_your_new_branch]
+```
 
 You then usually commit code changes, and push your branch back to GitHub like so:
 
@@ -50,9 +58,19 @@ A few tips:
 When you are ready to submit your pull-request:
 
 * Push your branch to your GitHub project.
-* Open a Pull Request on GitHub to the **rc (release candidate)** branch for a new feature or the **master** branch for a bug fix or documentation fix.
+* Open the pull request to the branch you've based your work on
+
+For more details on submitting a pull-request, please see:  [GitHub Guide to Collaborating with issues and pull requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests).
+
+## Branches within cBioPortal
+
+To figure out where your first pull request might go, it helps to have an understanding of cBioPortal's branching model. The cBioPortal currently maintains three branches in both the [frontend](https://github.com/cbioportal/cbioportal-frontend) and [backend repo](https://github.com/cbioportal/cbioportal):
 
-For instructions on submitting a pull-request, please see:  [Using Pull Requests ](https://help.github.com/articles/using-pull-requests/) and [Sending Pull Requests](http://help.github.com/send-pull-requests/).
+ * **master**:  this reflects what will be released with our next weekly release (https://github.com/cBioPortal/cbioportal/releases). For the [frontend repo](https://github.com/cbioportal/cbioportal-frontend) this branch is automatically deployed to production. On the backend it is deployed at least once a week. New features, bug fixes and documentation updates can go here. Only if the feature requires a database migration it should go to **rc**.
+  * **rc**:  release candidate branch, this branch contains new changes that require a database migration. It is deployed to https://rc.cbioportal.org. Once it's ready for more thorough product review a new branch is created with the name **release-x.y.z**. This way people can still continue to submit new changes to **rc**, while more thorough testing takes place of the **release-x.y.z** branch.
+  *  **release-x.y.z**: this branch contains changes that require a database migration. It will be merged to master after thorough product review on https://beta.cbioportal.org.
+
+We try to continuously merge new changes from `master` to `release-x.y.z`, and subsequently from the `release-x.y.z` branch to `rc` such that everybody is working on the latest code. Keep in mind though that occasionally there are conflicts that need to be resolved before we can merge. If you're working on e.g. the rc branch, you can check whether all the changes in master are in rc like this: https://github.com/cBioPortal/cbioportal/compare/rc...master. If a particular change you are waiting for is not there, one can help creating a pull request that merges these changes in. Try e.g. (if origin points to the cbioportal repo): `git fetch origin && git checkout origin/rc -b merge-master-to-rc && git merge origin/master`. Then resolve conflicts and push the branch `merge-master-to-rc` to your repo and open the PR.
 
 ### Getting your changes reviewed
 
@@ -65,7 +83,7 @@ If you have an idea who might be able to spot such issues
 in the parts of the code and functionality affected by your changes,
 notify them by requesting a review using the **Reviewers** menu
 to the right of the summary you just wrote
-and/or `@`-mentioning them in a comment.
+and/or `@`-mentioning them in a comment. Or reaching out them on [slack](https://slack.cbioportal.org).
 
 Reviewers may request you to rephrase or adjust things
 before they allow the changes to be integrated.
@@ -77,38 +95,6 @@ until the reviewers approve.
 Reviewers may request you to squash such amendment commits afterwards,
 or offer to push rewritten versions of your commits themselves.
 
-## Automated tests on Travis CI
-All Pull Requests are automatically tested on [Travis
-CI](https://travis-ci.org/cBioPortal/cbioportal/pull_requests). Currently there
-is a set of tests for the core module and a visual regression test that makes
-some screenshots and compares them to the ones stored in the repository.
-
-### What to do if the screenshot test fails
-When the screenshot test fails, it means that the screenshot taken from your
-instance of the portal differs from the screenshot stored in the repo.
-Copy+Paste the URL in the Travis CI log to view the image diff online. Further
-instructions are outlined on that page.
-
-If you prefer to compare the images locally, you need to first download the
-failing screenshot. The Travis CI log will show you where the image was
-uploaded on [clbin.com](https://clbin.com). First, download the image and
-replace the screenshot in the repo. For instance run in the root dir of
-cBioPortal:
-
-```bash
-curl 'https://clbin.com/[replace-with-clbin-image-from-log].png' > test/end-to-end/screenshots/[replace-with-image-from-repo].png
-```
-
-Then follow the steps outlined in [this blog post](http://www.akikoskinen.info/image-diffs-with-git/) to compare the 
-images locally. Run `git diff` from your repo to see the ImageMagick diff.
-
-Once you downloaded the images you do the following for each screenshot:
-
-- If the change in the screenshot is **undesired**, i.e. there is regression, you
-  should fix your PR.
-- If the change in the screenshot is **desired**, add the screenshot to the
-  repo, commit it and push it to your PR's branch.
-
 ## Pull Request Reviewers Guide
 If someone requests your review on a pull request,
 read the title and description and assign any other collaborators
@@ -158,12 +144,12 @@ New features:
   (Maven) config goes in the appriopriate `pom.xml` (root, `scripts/`, `portal/`, `core/`).
   Runtime (Spring) goes in `portal.properties`. Default values should be in `GlobalProperties.java`.
 - Non-stable configuration should be done through war overlays.
-- Is the configuration tested as part of Travis CI? It's not a necessity but be
+- Is the configuration tested as part of the CI tests? It's not a necessity but be
   aware that untested configuration will be tough to maintain.
 - Is there documentation on the proposed changes?
 
 ## Additional Resources
 
 * [cBioPortal Issue Tracker](https://github.com/cBioPortal/cbioportal/issues)
 * [General GitHub documentation](http://help.github.com/)
-* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
+* [GitHub Pull Request documentation](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests)