Skip to content

Commit

Permalink
feat: implement gitlab catalog backend (#44)
Browse files Browse the repository at this point in the history
* First pass at Gitlab catalog backend plugin

* Include so its easier to setup

* TSC ignores on Gitlab client import usage

* Run prettier

* Filter users for quicker load

* Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts

Co-authored-by: Zach Hammer <[email protected]>

* Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.test.ts

Co-authored-by: Zach Hammer <[email protected]>

* Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts

Co-authored-by: Zach Hammer <[email protected]>

* Last linting changes

* Prettier

* README update

* Typo

---------

Co-authored-by: Zach Hammer <[email protected]>
  • Loading branch information
bbckr and zhammer authored Mar 20, 2024
1 parent 804cc2d commit 7927e46
Show file tree
Hide file tree
Showing 8 changed files with 379 additions and 0 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ This SeatGeek Backstage Plugins Collection offers the following plugins:
- [plugins/slack-catalog-backend](plugins/slack-catalog-backend/)
- **AWS Catalog**: the AWS Catalog Plugin offers catalog integrations with the AWS API.
- [plugins/aws-catalog-backend](plugins/aws-catalog-backend)
- **Gitlab Catalog**: the Gitlab Catalog Plugin offers catalog integrations with the Gitlab API.
- [plugins/gitlab-catalog-backend](plugins/gitlab-catalog-backend/)

Each of the plugins contain instructions for installation and development within
their respective locations.
Expand Down
7 changes: 7 additions & 0 deletions app-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,10 @@ catalog:
target: ../../mock-catalog/templates/create-postgres-database.yaml
- type: file
target: ../../mock-catalog/templates/create-python-module.yaml

slackCatalog:
token: ${SLACK_API_TOKEN_CATALOG}

gitlabCatalog:
host: ${GITLAB_HOST_CATALOG}
token: ${GITLAB_TOKEN_CATALOG}
1 change: 1 addition & 0 deletions plugins/gitlab-catalog-backend/.eslintrc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
51 changes: 51 additions & 0 deletions plugins/gitlab-catalog-backend/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# @seatgeek/backstage-plugin-gitlab-catalog-backend

This plugin offers catalog integrations for ingesting data from the Slack API into the Software Catalog.

[![npm latest version](https://img.shields.io/npm/v/@seatgeek/backstage-plugin-gitlab-catalog-backend/latest.svg)](https://www.npmjs.com/package/@seatgeek/backstage-plugin-gitlab-catalog-backend)

## Installation

Install the `@seatgeek/backstage-plugin-gitlab-catalog-backend` package in your backend package:

```shell
# From your Backstage root directory
yarn add --cwd packages/backend @seatgeek/backstage-plugin-gitlab-catalog-backend
```

Add the following config to your `app-config.yaml`:

```yml
gitlabCatalog:
host: ${GITLAB_HOST_CATALOG} # defaults to https://gitlab.com
token: ${GITLAB_TOKEN_CATALOG}
```
Requires `read_user` scope with administrator level permissions to be able to view the email, see [List Users (for administrators)](https://docs.gitlab.com/ee/api/users.html#for-administrators).

## Processors

### `GitlabUserProcessor`

Enriches existing `User` entities with information from Gitlab, notably the user's Gitlab ID, based on the user's `.profile.email`.

#### Installation

Add the following to your `packages/backend/catalog.ts`:

```ts
import { GitlabUserProcessor } from '@seatgeek/backstage-plugin-gitlab-catalog-backend';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = CatalogBuilder.create(env);
builder.addProcessor(
// Add the gitlab user processor
GitlabUserProcessor.fromConfig(env.config, env.logger),
);
const { processingEngine, router } = await builder.build();
processingEngine.start();
return router;
}
```
46 changes: 46 additions & 0 deletions plugins/gitlab-catalog-backend/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
"name": "@seatgeek/backstage-plugin-gitlab-catalog-backend",
"version": "0.0.0-semantically-released",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin"
},
"scripts": {
"start": "backstage-cli package start",
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"clean": "backstage-cli package clean",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@backstage/backend-common": "^0.20.1",
"@backstage/catalog-model": "^1.4.3",
"@backstage/config": "^1.1.1",
"@backstage/plugin-catalog-common": "^1.0.20",
"@gitbeaker/rest": "^40.0.1",
"@types/express": "*",
"express": "^4.17.1",
"express-promise-router": "^4.1.0",
"node-fetch": "^2.6.7",
"winston": "^3.2.1",
"yn": "^4.0.0"
},
"devDependencies": {
"@backstage/cli": "^0.25.1",
"@types/supertest": "^2.0.12",
"msw": "^1.0.0",
"supertest": "^6.2.4"
},
"files": [
"dist"
]
}
119 changes: 119 additions & 0 deletions plugins/gitlab-catalog-backend/src/GitlabUserProcessor.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Copyright SeatGeek
* Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms.
*/
import { SystemEntity, UserEntity } from '@backstage/catalog-model';
import { Gitlab } from '@gitbeaker/rest';
import * as winston from 'winston';
import { GitlabUserProcessor } from './GitlabUserProcessor';

jest.mock('@gitbeaker/rest', () => {
return {
Gitlab: jest.fn().mockImplementation(() => {
return {
Users: {
all: jest.fn().mockResolvedValue([
{
id: 123,
email: '[email protected]',
},
{
id: 999,
email: '[email protected]',
},
]),
// mock other methods as needed
},
};
}),
};
});

describe('GitlabUserProcessor', () => {
let processor: GitlabUserProcessor;
let mockGitlabClient: InstanceType<typeof Gitlab>;
let mockLogger: winston.Logger;

beforeEach(() => {
mockGitlabClient = new Gitlab({ token: `token` });
mockLogger = winston.createLogger({
transports: [
new winston.transports.Console({
format: winston.format.simple(),
level: 'debug',
}),
],
});

processor = new GitlabUserProcessor(mockGitlabClient, mockLogger);

// Reset the mocks before each test
jest.clearAllMocks();
});

test('should add gitlab info', async () => {
const before: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'rufus',
},
spec: {
profile: {
email: '[email protected]',
},
},
};
const result = await processor.postProcessEntity(
before,
{} as any,
() => {},
);

const expected: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'rufus',
annotations: {
'gitlab.com/user_id': '123',
},
},
spec: {
profile: {
email: '[email protected]',
},
},
};

expect(result).toEqual(expected);
expect(mockGitlabClient.Users.all).toHaveBeenCalled();

// make sure that the slack users are only fetched once
await processor.postProcessEntity(before, {} as any, () => {});
expect(mockGitlabClient.Users.all).toHaveBeenCalledTimes(1);
});

test('should no op if not a user', async () => {
const before: SystemEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'System',
metadata: {
name: 'rufus',
annotations: {},
},
spec: {
owner: 'rufus',
},
};

const result = await processor.postProcessEntity(
before,
{} as any,
() => {},
);

expect(result).toEqual(before);
expect(mockGitlabClient.Users.all).not.toHaveBeenCalled();
});
});
148 changes: 148 additions & 0 deletions plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/*
* Copyright SeatGeek
* Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms.
*/
import { Entity, isUserEntity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { LocationSpec } from '@backstage/plugin-catalog-common';
import type {
CatalogProcessor,
CatalogProcessorEmit,
} from '@backstage/plugin-catalog-node';
import { ExpandedUserSchema, Gitlab } from '@gitbeaker/rest';
import { Logger } from 'winston';

const GITLAB_PER_PAGE_LIMIT = 500;
const GITLAB_DEFAULT_HOST = 'https://gitlab.com';

/**
* The GitlabUserProcessor is used to enrich our User entities with information
* from Gitlab: notably the user's Gitlab ID.
*
* @public
*/
export class GitlabUserProcessor implements CatalogProcessor {
private readonly gitlab: InstanceType<typeof Gitlab>;
private readonly logger: Logger;
private cacheLoaded: boolean;
private userLookup: Map<string, ExpandedUserSchema>;
// guarantee that users are loaded only once
private loadUserPromise: Promise<Map<string, ExpandedUserSchema>> | null =
null;

private async fetchUsers(): Promise<Map<string, ExpandedUserSchema>> {
if (!this.gitlab) {
return new Map();
}

if (!this.cacheLoaded) {
// we use a shared promise to make sure that the load is only
// executed once even if `GitlabUserProcessor.postProcessEntity`
// is called "concurrently".
if (!this.loadUserPromise) {
this.loadUserPromise = this.loadUsers();
}

// Wait for the data to load
await this.loadUserPromise;
}

return this.userLookup;
}

private async loadUsers(): Promise<Map<string, ExpandedUserSchema>> {
this.logger.info('Loading gitlab users');

let members: ExpandedUserSchema[] = [];
try {
members = (await this.gitlab!.Users.all({
perPage: GITLAB_PER_PAGE_LIMIT,
active: true,
withoutProjectBots: true,
})) as ExpandedUserSchema[];
} catch (error) {
this.logger.error(`Error loading gitlab users: ${error}`);
return this.userLookup;
}

members.forEach(user => {
if (user.email) {
this.userLookup.set(user.email, user);
}
});

this.logger.info(`Loaded ${this.userLookup.size} gitlab users`);
this.cacheLoaded = true;

return this.userLookup;
}

static fromConfig(config: Config, logger: Logger): GitlabUserProcessor[] {
const token = config.getOptionalString('gitlabCatalog.token');
if (!token) {
logger.warn(
'No token provided for GitlabUserProcessor, skipping Gitlab user lookup',
);
return [];
}
let host = config.getOptionalString('gitlabCatalog.host');
if (!host) {
logger.info(
`No host provided for GitlabUserProcessor, defaulting to ${GITLAB_DEFAULT_HOST}`,
);
host = GITLAB_DEFAULT_HOST;
}
return [
new GitlabUserProcessor(
new Gitlab({
host: host,
token: token,
}),
logger,
),
];
}

constructor(gitlab: InstanceType<typeof Gitlab>, logger: Logger) {
this.gitlab = gitlab;
this.logger = logger;
this.userLookup = new Map();
this.cacheLoaded = false;
}

getProcessorName(): string {
return 'GitlabUserProcessor';
}

async postProcessEntity(
entity: Entity,
_location: LocationSpec,
_emit: CatalogProcessorEmit,
): Promise<Entity> {
if (!isUserEntity(entity)) {
return entity;
}

const email = entity.spec.profile?.email;
if (!email) {
return entity;
}

const userLookup = await this.fetchUsers();
const gitlabUser = userLookup.get(email);
if (!gitlabUser) {
return entity;
}

if (!entity.metadata.annotations) {
entity.metadata.annotations = {};
}

if (gitlabUser.id) {
entity.metadata.annotations[`gitlab.com/user_id`] =
gitlabUser.id.toString();
}

return entity;
}
}
5 changes: 5 additions & 0 deletions plugins/gitlab-catalog-backend/src/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
/*
* Copyright SeatGeek
* Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms.
*/
export { GitlabUserProcessor } from './GitlabUserProcessor';

0 comments on commit 7927e46

Please sign in to comment.