diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..169a724 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,6 @@ +docker-compose.yml +build_info.txt +prometheus/ + +## python specific +*.pyc diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..b675662 --- /dev/null +++ b/src/README.md @@ -0,0 +1 @@ +# Micorservices diff --git a/src/comment/Dockerfile b/src/comment/Dockerfile new file mode 100644 index 0000000..7f2520e --- /dev/null +++ b/src/comment/Dockerfile @@ -0,0 +1,14 @@ +FROM ruby:2.2 +RUN apt-get update -qq && apt-get install -y build-essential + +ENV APP_HOME /app +RUN mkdir $APP_HOME +WORKDIR $APP_HOME +ADD Gemfile* $APP_HOME/ +RUN bundle install +COPY . $APP_HOME + +ENV COMMENT_DATABASE_HOST comment_db +ENV COMMENT_DATABASE comments +CMD ["puma"] + diff --git a/src/comment/Gemfile b/src/comment/Gemfile new file mode 100644 index 0000000..82e3ac2 --- /dev/null +++ b/src/comment/Gemfile @@ -0,0 +1,10 @@ +source 'https://rubygems.org' + +gem 'sinatra', '~> 2.0.2' +gem 'bson_ext' +gem 'mongo' +gem 'puma' +gem 'prometheus-client' +gem "rack", '>= 2.0.6' +gem 'rufus-scheduler' +gem 'tzinfo-data' diff --git a/src/comment/Gemfile.lock b/src/comment/Gemfile.lock new file mode 100644 index 0000000..e095838 --- /dev/null +++ b/src/comment/Gemfile.lock @@ -0,0 +1,50 @@ +GEM + remote: https://rubygems.org/ + specs: + bson (4.3.0) + bson_ext (1.5.1) + et-orbi (1.1.6) + tzinfo + fugit (1.1.6) + et-orbi (~> 1.1, >= 1.1.6) + raabro (~> 1.1) + mongo (2.6.2) + bson (>= 4.3.0, < 5.0.0) + mustermann (1.0.3) + prometheus-client (0.8.0) + quantile (~> 0.2.1) + puma (3.12.0) + quantile (0.2.1) + raabro (1.1.6) + rack (2.0.6) + rack-protection (2.0.4) + rack + rufus-scheduler (3.5.2) + fugit (~> 1.1, >= 1.1.5) + sinatra (2.0.4) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.4) + tilt (~> 2.0) + thread_safe (0.3.6) + tilt (2.0.9) + tzinfo (1.2.5) + thread_safe (~> 0.1) + tzinfo-data (1.2018.7) + tzinfo (>= 1.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + bson_ext + mongo + prometheus-client + puma + rack (>= 2.0.6) + rufus-scheduler + sinatra (~> 2.0.2) + tzinfo-data + +BUNDLED WITH + 1.17.2 diff --git a/src/comment/VERSION b/src/comment/VERSION new file mode 100644 index 0000000..bcab45a --- /dev/null +++ b/src/comment/VERSION @@ -0,0 +1 @@ +0.0.3 diff --git a/src/comment/comment_app.rb b/src/comment/comment_app.rb new file mode 100644 index 0000000..942e9a0 --- /dev/null +++ b/src/comment/comment_app.rb @@ -0,0 +1,132 @@ +require 'sinatra' +require 'json/ext' +require 'uri' +require 'mongo' +require 'prometheus/client' +require 'rufus-scheduler' +require_relative 'helpers' + +# Database connection info +COMMENT_DATABASE_HOST ||= ENV['COMMENT_DATABASE_HOST'] || '127.0.0.1' +COMMENT_DATABASE_PORT ||= ENV['COMMENT_DATABASE_PORT'] || '27017' +COMMENT_DATABASE ||= ENV['COMMENT_DATABASE'] || 'test' +DB_URL ||= "mongodb://#{COMMENT_DATABASE_HOST}:#{COMMENT_DATABASE_PORT}" + +# App version and build info +if File.exist?('VERSION') + VERSION ||= File.read('VERSION').strip +else + VERSION ||= "version_missing" +end + +if File.exist?('build_info.txt') + BUILD_INFO = File.readlines('build_info.txt') +else + BUILD_INFO = Array.new(2, "build_info_missing") +end + +configure do + Mongo::Logger.logger.level = Logger::WARN + db = Mongo::Client.new(DB_URL, database: COMMENT_DATABASE, + heartbeat_frequency: 2) + set :mongo_db, db[:comments] + set :bind, '0.0.0.0' + set :server, :puma + set :logging, false + set :mylogger, Logger.new(STDOUT) +end + +# Create and register metrics +prometheus = Prometheus::Client.registry +comment_health_gauge = Prometheus::Client::Gauge.new( + :comment_health, + 'Health status of Comment service' +) +comment_health_db_gauge = Prometheus::Client::Gauge.new( + :comment_health_mongo_availability, + 'Check if MongoDB is available to Comment' +) +comment_count = Prometheus::Client::Counter.new( + :comment_count, + 'A counter of new comments' +) +prometheus.register(comment_health_gauge) +prometheus.register(comment_health_db_gauge) +prometheus.register(comment_count) + +# Schedule health check function +scheduler = Rufus::Scheduler.new +scheduler.every '5s' do + check = JSON.parse(healthcheck_handler(DB_URL, VERSION)) + set_health_gauge(comment_health_gauge, check['status']) + set_health_gauge(comment_health_db_gauge, check['dependent_services']['commentdb']) +end + +before do + env['rack.logger'] = settings.mylogger # set custom logger +end + +after do + request_id = env['HTTP_REQUEST_ID'] || 'null' + logger.info('service=comment | event=request | ' \ + "path=#{env['REQUEST_PATH']}\n" \ + "request_id=#{request_id} | " \ + "remote_addr=#{env['REMOTE_ADDR']} | " \ + "method= #{env['REQUEST_METHOD']} | " \ + "response_status=#{response.status}") +end + +# retrieve post's comments +get '/:id/comments' do + id = obj_id(params[:id]) + begin + posts = settings.mongo_db.find(post_id: id.to_s).to_a.to_json + rescue StandardError => e + log_event('error', 'find_post_comments', + "Couldn't retrieve comments from DB. Reason: #{e.message}", + params) + halt 500 + else + log_event('info', 'find_post_comments', + 'Successfully retrieved post comments from DB', params) + posts + end +end + +# add a new comment +post '/add_comment/?' do + begin + prms = { post_id: params['post_id'], + name: params['name'], + email: params['email'], + body: params['body'], + created_at: params['created_at'] } + rescue StandardError => e + log_event('error', 'add_comment', + "Bat input data. Reason: #{e.message}", prms) + end + db = settings.mongo_db + begin + result = db.insert_one post_id: params['post_id'], name: params['name'], + email: params['email'], body: params['body'], + created_at: params['created_at'] + db.find(_id: result.inserted_id).to_a.first.to_json + rescue StandardError => e + log_event('error', 'add_comment', + "Failed to create a comment. Reason: #{e.message}", params) + halt 500 + else + log_event('info', 'add_comment', + 'Successfully created a new comment', params) + comment_count.increment + end +end + +# health check endpoint +get '/healthcheck' do + healthcheck_handler(DB_URL, VERSION) +end + +get '/*' do + halt 404, 'Page not found' +end diff --git a/src/comment/config.ru b/src/comment/config.ru new file mode 100644 index 0000000..5f17c56 --- /dev/null +++ b/src/comment/config.ru @@ -0,0 +1,10 @@ +require './comment_app' +require 'rack' +require 'prometheus/middleware/collector' +require 'prometheus/middleware/exporter' + +use Rack::Deflater, if: ->(_, _, _, body) { body.any? && body[0].length > 512 } +use Prometheus::Middleware::Collector +use Prometheus::Middleware::Exporter + +run Sinatra::Application diff --git a/src/comment/docker_build.sh b/src/comment/docker_build.sh new file mode 100644 index 0000000..a4c8e82 --- /dev/null +++ b/src/comment/docker_build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo `git show --format="%h" HEAD | head -1` > build_info.txt +echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt + +docker build -t $USER_NAME/comment . diff --git a/src/comment/helpers.rb b/src/comment/helpers.rb new file mode 100644 index 0000000..0b1e47b --- /dev/null +++ b/src/comment/helpers.rb @@ -0,0 +1,75 @@ +def obj_id(val) + begin + BSON::ObjectId.from_string(val) + rescue BSON::ObjectId::Invalid + nil + end +end + +def document_by_id(id) + id = obj_id(id) if String === id + if id.nil? + {}.to_json + else + document = settings.mongo_db.find(_id: id).to_a.first + (document || {}).to_json + end +end + +def healthcheck_handler(db_url, version) + begin + commentdb_test = Mongo::Client.new(db_url, + server_selection_timeout: 2) + commentdb_test.database_names + commentdb_test.close + rescue StandardError + commentdb_status = 0 + else + commentdb_status = 1 + end + + status = commentdb_status + healthcheck = { + status: status, + dependent_services: { + commentdb: commentdb_status + }, + version: version + } + + healthcheck.to_json +end + +def set_health_gauge(metric, value) + metric.set( + { + version: VERSION, + commit_hash: BUILD_INFO[0].strip, + branch: BUILD_INFO[1].strip + }, + value + ) +end + +def log_event(type, name, message, params = '{}') + case type + when 'error' + logger.error('service=comment | ' \ + "event=#{name} | " \ + "request_id=#{request.env['HTTP_REQUEST_ID']}\n" \ + "message=\'#{message}\'\n" \ + "params: #{params.to_json}") + when 'info' + logger.info('service=comment | ' \ + "event=#{name} | " \ + "request_id=#{request.env['HTTP_REQUEST_ID']}\n" \ + "message=\'#{message}\'\n" \ + "params: #{params.to_json}") + when 'warning' + logger.warn('service=comment | ' \ + "event=#{name} | " \ + "request_id=#{request.env['HTTP_REQUEST_ID']}\n" \ + "message=\'#{message}\'\n" \ + "params: #{params.to_json}") + end +end diff --git a/src/post-py/Dockerfile b/src/post-py/Dockerfile new file mode 100644 index 0000000..c756c74 --- /dev/null +++ b/src/post-py/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.6.0-alpine + +WORKDIR /app +ADD . /app/ + +RUN apk update +RUN apk add gcc +RUN apk add musl-dev + + +RUN pip install --upgrade pip +RUN pip install wheel +RUN pip install -r /app/requirements.txt + + +ENV POST_DATABASE_HOST post_db +ENV POST_DATABASE posts + +CMD ["python3", "post_app.py"] + diff --git a/src/post-py/VERSION b/src/post-py/VERSION new file mode 100644 index 0000000..4e379d2 --- /dev/null +++ b/src/post-py/VERSION @@ -0,0 +1 @@ +0.0.2 diff --git a/src/post-py/docker_build.sh b/src/post-py/docker_build.sh new file mode 100644 index 0000000..ccce114 --- /dev/null +++ b/src/post-py/docker_build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo `git show --format="%h" HEAD | head -1` > build_info.txt +echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt + +docker build -t $USER_NAME/post . diff --git a/src/post-py/helpers.py b/src/post-py/helpers.py new file mode 100644 index 0000000..5c64941 --- /dev/null +++ b/src/post-py/helpers.py @@ -0,0 +1,40 @@ +import structlog +from pymongo import MongoClient +from pymongo.errors import ConnectionFailure +from json import dumps +from flask import request + + +log = structlog.get_logger() + + +def http_healthcheck_handler(mongo_host, mongo_port, version): + postdb = MongoClient(mongo_host, int(mongo_port), + serverSelectionTimeoutMS=2000) + try: + postdb.admin.command('ismaster') + except ConnectionFailure: + postdb_status = 0 + else: + postdb_status = 1 + + status = postdb_status + healthcheck = { + 'status': status, + 'dependent_services': { + 'postdb': postdb_status + }, + 'version': version + } + return dumps(healthcheck) + + +def log_event(event_type, name, message, params={}): + request_id = request.headers['Request-Id'] \ + if 'Request-Id' in request.headers else None + if event_type == 'info': + log.info(name, service='post', request_id=request_id, + message=message, params=params) + elif event_type == 'error': + log.error(name, service='post', request_id=request_id, + message=message, params=params) diff --git a/src/post-py/post_app.py b/src/post-py/post_app.py new file mode 100644 index 0000000..8c1ead8 --- /dev/null +++ b/src/post-py/post_app.py @@ -0,0 +1,277 @@ +import os +import prometheus_client +import time +import structlog +import traceback +import requests +from flask import Flask, request, Response, abort, logging +from pymongo import MongoClient +from bson.objectid import ObjectId +from bson.json_util import dumps +from helpers import http_healthcheck_handler, log_event +from py_zipkin.zipkin import zipkin_span, ZipkinAttrs +from py_zipkin.transport import BaseTransportHandler + + +CONTENT_TYPE_LATEST = str('text/plain; version=0.0.4; charset=utf-8') +POST_DATABASE_HOST = os.getenv('POST_DATABASE_HOST', '127.0.0.1') +POST_DATABASE_PORT = os.getenv('POST_DATABASE_PORT', '27017') +ZIPKIN_HOST = os.getenv('ZIPKIN_HOST', 'zipkin') +ZIPKIN_PORT = os.getenv('ZIPKIN_PORT', '9411') +ZIPKIN_URL = "http://{0}:{1}/api/v1/spans".format(ZIPKIN_HOST, ZIPKIN_PORT) +ZIPKIN_ENABLED = bool(os.getenv('ZIPKIN_ENABLED', False)) + +log = structlog.get_logger() + +app = Flask(__name__) + +class DummyTransport(BaseTransportHandler): + + def get_max_payload_bytes(self): + return None + + def send(self, encoded_span): + # The collector does nothing at all + return None + +class HttpTransport(BaseTransportHandler): + + def get_max_payload_bytes(self): + return None + + def send(self, encoded_span): + # The collector expects a thrift-encoded list of spans. Instead of + # decoding and re-encoding the already thrift-encoded message, we can just + # add header bytes that specify that what follows is a list of length 1. + body = b'\x0c\x00\x00\x00\x01' + encoded_span + try: + requests.post(ZIPKIN_URL, data=body, + headers={'Content-Type': 'application/x-thrift'}) + except requests.exceptions.RequestException: + tb = traceback.format_exc() + log.error('zipkin_error', + service='post', + traceback=tb) + +zipkin_transport = HttpTransport() \ + if ZIPKIN_ENABLED else DummyTransport() + +if ZIPKIN_ENABLED: + def zipkin_fill_attrs(headers): + try: + zipkin_attrs=ZipkinAttrs( + trace_id=headers['X-B3-TraceID'], + span_id=headers['X-B3-SpanID'], + parent_span_id=headers['X-B3-ParentSpanID'], + flags=headers['X-B3-Flags'], + is_sampled=headers['X-B3-Sampled'], + ) + except KeyError: + log_event('warning', 'zipkin_fill_headers', "Tracing enabled and some Zipkin HTTP headers are missing") + zipkin_attrs = None + pass + return zipkin_attrs +else: + def zipkin_fill_attrs(headers): + return None + +def init(app): + # application version info + app.version = None + with open('VERSION') as f: + app.version = f.read().rstrip() + + # prometheus metrics + app.post_read_db_seconds = prometheus_client.Histogram( + 'post_read_db_seconds', + 'Request DB time' + ) + app.post_count = prometheus_client.Counter( + 'post_count', + 'A counter of new posts' + ) + # database client connection + app.db = MongoClient( + POST_DATABASE_HOST, + int(POST_DATABASE_PORT) + ).users_post.posts + + +# Prometheus endpoint +@app.route('/metrics') +def metrics(): + return Response(prometheus_client.generate_latest(), + mimetype=CONTENT_TYPE_LATEST) + + +# Retrieve information about all posts +@zipkin_span(service_name='post', span_name='db_find_all_posts') +def find_posts(): + try: + posts = app.db.find().sort('created_at', -1) + except Exception as e: + log_event('error', 'find_all_posts', + "Failed to retrieve posts from the database. \ + Reason: {}".format(str(e))) + abort(500) + else: + log_event('info', 'find_all_posts', + 'Successfully retrieved all posts from the database') + return dumps(posts) + + +@app.route("/posts") +def posts(): + with zipkin_span( + service_name='post', + zipkin_attrs=zipkin_fill_attrs(request.headers), + span_name='/posts', + transport_handler=zipkin_transport, + port=5000, + sample_rate=100, + ): + posts = find_posts() + return posts + + +# Vote for a post +@app.route('/vote', methods=['POST']) +def vote(): + try: + post_id = request.values.get('id') + vote_type = request.values.get('type') + except Exception as e: + log_event('error', 'request_error', + "Bad input parameters. Reason: {}".format(str(e))) + abort(400) + try: + post = app.db.find_one({'_id': ObjectId(post_id)}) + post['votes'] += int(vote_type) + app.db.update_one({'_id': ObjectId(post_id)}, + {'$set': {'votes': post['votes']}}) + except Exception as e: + log_event('error', 'post_vote', + "Failed to vote for a post. Reason: {}".format(str(e)), + {'post_id': post_id, 'vote_type': vote_type}) + abort(500) + else: + log_event('info', 'post_vote', 'Successful vote', + {'post_id': post_id, 'vote_type': vote_type}) + return 'OK' + + +# Add new post +@app.route('/add_post', methods=['POST']) +def add_post(): + try: + title = request.values.get('title') + link = request.values.get('link') + created_at = request.values.get('created_at') + except Exception as e: + log_event('error', 'request_error', + "Bad input parameters. Reason: {}".format(str(e))) + abort(400) + try: + app.db.insert({'title': title, 'link': link, + 'created_at': created_at, 'votes': 0}) + except Exception as e: + log_event('error', 'post_create', + "Failed to create a post. Reason: {}".format(str(e)), + {'title': title, 'link': link}) + abort(500) + else: + log_event('info', 'post_create', 'Successfully created a new post', + {'title': title, 'link': link}) + app.post_count.inc() + return 'OK' + + +# Retrieve information about a post +@zipkin_span(service_name='post', span_name='db_find_single_post') +def find_post(id): + start_time = time.time() + try: + post = app.db.find_one({'_id': ObjectId(id)}) + except Exception as e: + log_event('error', 'post_find', + "Failed to find the post. Reason: {}".format(str(e)), + request.values) + abort(500) + else: + stop_time = time.time() # + 0.3 + resp_time = stop_time - start_time + app.post_read_db_seconds.observe(resp_time) + log_event('info', 'post_find', + 'Successfully found the post information', + {'post_id': id}) + return dumps(post) + + +# Find a post +@app.route('/post/') +def get_post(id): + with zipkin_span( + service_name='post', + zipkin_attrs=zipkin_fill_attrs(request.headers), + span_name='/post/', + transport_handler=zipkin_transport, + port=5000, + sample_rate=100, + ): + post = find_post(id) + return post + + +# Health check endpoint +@app.route('/healthcheck') +def healthcheck(): + return http_healthcheck_handler(POST_DATABASE_HOST, + POST_DATABASE_PORT, + app.version) + + +# Log every request +@app.after_request +def after_request(response): + request_id = request.headers['Request-Id'] \ + if 'Request-Id' in request.headers else None + log.info('request', + service='post', + request_id=request_id, + path=request.full_path, + addr=request.remote_addr, + method=request.method, + response_status=response.status_code) + return response + + +# Log Exceptions +@app.errorhandler(Exception) +def exceptions(e): + request_id = request.headers['Request-Id'] \ + if 'Request-Id' in request.headers else None + tb = traceback.format_exc() + log.error('internal_error', + service='post', + request_id=request_id, + path=request.full_path, + remote_addr=request.remote_addr, + method=request.method, + traceback=tb) + return 'Internal Server Error', 500 + + +if __name__ == "__main__": + init(app) + logg = logging.getLogger('werkzeug') + logg.disabled = True # disable default logger + # define log structure + structlog.configure(processors=[ + structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S"), + structlog.stdlib.add_log_level, + # to see indented logs in the terminal, uncomment the line below + # structlog.processors.JSONRenderer(indent=2, sort_keys=True) + # and comment out the one below + structlog.processors.JSONRenderer(sort_keys=True) + ]) + app.run(host='0.0.0.0', debug=True) diff --git a/src/post-py/requirements.txt b/src/post-py/requirements.txt new file mode 100644 index 0000000..d9bdc00 --- /dev/null +++ b/src/post-py/requirements.txt @@ -0,0 +1,6 @@ +prometheus_client==0.0.21 +flask==0.12.3 +pymongo==3.5.1 +structlog==17.2.0 +py-zipkin==0.13.0 +requests==2.18.4 diff --git a/src/ui/Dockerfile b/src/ui/Dockerfile new file mode 100644 index 0000000..00f9f8f --- /dev/null +++ b/src/ui/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:16.04 +RUN apt-get update \ + && apt-get install -y ruby-full ruby-dev build-essential \ + && gem install bundler --no-ri --no-rdoc + +ENV APP_HOME /app +RUN mkdir $APP_HOME + +WORKDIR $APP_HOME +ADD Gemfile* $APP_HOME/ +RUN bundle install +ADD . $APP_HOME + +ENV POST_SERVICE_HOST post +ENV POST_SERVICE_PORT 5000 +ENV COMMENT_SERVICE_HOST comment +ENV COMMENT_SERVICE_PORT 9292 + +CMD ["puma"] + diff --git a/src/ui/Gemfile b/src/ui/Gemfile new file mode 100644 index 0000000..dfc92d5 --- /dev/null +++ b/src/ui/Gemfile @@ -0,0 +1,13 @@ +source 'https://rubygems.org' + +gem 'sinatra', '~> 2.0.2' +gem 'sinatra-contrib' +gem 'haml' +gem 'bson_ext' +gem 'faraday' +gem 'puma' +gem 'prometheus-client' +gem "rack", '>= 2.0.6' +gem 'rufus-scheduler' +gem 'tzinfo-data' +gem 'zipkin-tracer' diff --git a/src/ui/Gemfile.lock b/src/ui/Gemfile.lock new file mode 100644 index 0000000..d5d5400 --- /dev/null +++ b/src/ui/Gemfile.lock @@ -0,0 +1,88 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (5.2.2) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + backports (3.11.4) + bson (1.12.5) + bson_ext (1.12.5) + bson (~> 1.12.5) + concurrent-ruby (1.1.4) + et-orbi (1.1.6) + tzinfo + faraday (0.15.4) + multipart-post (>= 1.2, < 3) + finagle-thrift (1.4.2) + thrift (~> 0.9.3) + fugit (1.1.6) + et-orbi (~> 1.1, >= 1.1.6) + raabro (~> 1.1) + haml (5.0.4) + temple (>= 0.8.0) + tilt + i18n (1.3.0) + concurrent-ruby (~> 1.0) + minitest (5.11.3) + multi_json (1.13.1) + multipart-post (2.0.0) + mustermann (1.0.3) + prometheus-client (0.8.0) + quantile (~> 0.2.1) + puma (3.12.0) + quantile (0.2.1) + raabro (1.1.6) + rack (2.0.6) + rack-protection (2.0.4) + rack + rufus-scheduler (3.5.2) + fugit (~> 1.1, >= 1.1.5) + sinatra (2.0.4) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.4) + tilt (~> 2.0) + sinatra-contrib (2.0.4) + activesupport (>= 4.0.0) + backports (>= 2.8.2) + multi_json + mustermann (~> 1.0) + rack-protection (= 2.0.4) + sinatra (= 2.0.4) + tilt (>= 1.3, < 3) + sucker_punch (2.1.1) + concurrent-ruby (~> 1.0) + temple (0.8.0) + thread_safe (0.3.6) + thrift (0.9.3.0) + tilt (2.0.9) + tzinfo (1.2.5) + thread_safe (~> 0.1) + tzinfo-data (1.2018.7) + tzinfo (>= 1.0.0) + zipkin-tracer (0.30.0) + faraday (~> 0.8) + finagle-thrift (~> 1.4.2) + rack (>= 1.0) + sucker_punch (~> 2.0) + +PLATFORMS + ruby + +DEPENDENCIES + bson_ext + faraday + haml + prometheus-client + puma + rack (>= 2.0.6) + rufus-scheduler + sinatra (~> 2.0.2) + sinatra-contrib + tzinfo-data + zipkin-tracer + +BUNDLED WITH + 1.17.2 diff --git a/src/ui/VERSION b/src/ui/VERSION new file mode 100644 index 0000000..8acdd82 --- /dev/null +++ b/src/ui/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/src/ui/config.ru b/src/ui/config.ru new file mode 100644 index 0000000..92cb1ee --- /dev/null +++ b/src/ui/config.ru @@ -0,0 +1,14 @@ +require 'bundler/setup' +require 'rack' +require 'prometheus/middleware/collector' +require 'prometheus/middleware/exporter' +require_relative 'ui_app' +require_relative 'middleware' +require 'zipkin-tracer' + +use Metrics +use Rack::Deflater, if: ->(_, _, _, body) { body.any? && body[0].length > 512 } +use Prometheus::Middleware::Collector +use Prometheus::Middleware::Exporter + +run Sinatra::Application diff --git a/src/ui/docker_build.sh b/src/ui/docker_build.sh new file mode 100644 index 0000000..5dd1c0e --- /dev/null +++ b/src/ui/docker_build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo `git show --format="%h" HEAD | head -1` > build_info.txt +echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt + +docker build -t $USER_NAME/ui . diff --git a/src/ui/helpers.rb b/src/ui/helpers.rb new file mode 100644 index 0000000..dc94d0a --- /dev/null +++ b/src/ui/helpers.rb @@ -0,0 +1,82 @@ +def flash_danger(message) + session[:flashes] << { type: 'alert-danger', message: message } +end + +def flash_success(message) + session[:flashes] << { type: 'alert-success', message: message } +end + +def log_event(type, name, message, params = '{}') + case type + when 'error' + logger.error('service=ui | ' \ + "event=#{name} | " \ + "request_id=#{request.env['REQUEST_ID']} | " \ + "message=\'#{message}\' | " \ + "params: #{params.to_json}") + when 'info' + logger.info('service=ui | ' \ + "event=#{name} | " \ + "request_id=#{request.env['REQUEST_ID']} | " \ + "message=\'#{message}\' | " \ + "params: #{params.to_json}") + when 'warning' + logger.warn('service=ui | ' \ + "event=#{name} | " \ + "request_id=#{request.env['REQUEST_ID']} | " \ + "message=\'#{message}\' | " \ + "params: #{params.to_json}") + end +end + +def http_request(method, url, params = {}) + unless defined?(request).nil? + settings.http_client.headers[:request_id] = request.env['REQUEST_ID'].to_s + end + + case method + when 'get' + response = settings.http_client.get url + JSON.parse(response.body) + when 'post' + settings.http_client.post url, params + end +end + +def http_healthcheck_handler(post_url, comment_url, version) + post_status = check_service_health(post_url) + comment_status = check_service_health(comment_url) + + status = if comment_status == 1 && post_status == 1 + 1 + else + 0 + end + + healthcheck = { status: status, + dependent_services: { + comment: comment_status, + post: post_status + }, + version: version } + healthcheck.to_json +end + +def check_service_health(url) + name = http_request('get', "#{url}/healthcheck") +rescue StandardError + 0 +else + name['status'] +end + +def set_health_gauge(metric, value) + metric.set( + { + version: VERSION, + commit_hash: BUILD_INFO[0].strip, + branch: BUILD_INFO[1].strip + }, + value + ) +end diff --git a/src/ui/middleware.rb b/src/ui/middleware.rb new file mode 100644 index 0000000..8f70cca --- /dev/null +++ b/src/ui/middleware.rb @@ -0,0 +1,32 @@ +require 'prometheus/client' + +class Metrics + def initialize(app) + @app = app + prometheus = Prometheus::Client.registry + @request_count = Prometheus::Client::Counter.new( + :ui_request_count, + 'App Request Count' + ) + @request_response_time = Prometheus::Client::Histogram.new( + :ui_request_response_time, + 'Request response time' + ) + prometheus.register(@request_response_time) + prometheus.register(@request_count) + end + + def call(env) + request_started_on = Time.now + env['REQUEST_ID'] = SecureRandom.uuid # add unique ID to each request + @status, @headers, @response = @app.call(env) + request_ended_on = Time.now + # prometheus metrics + @request_response_time.observe({ path: env['REQUEST_PATH'] }, + request_ended_on - request_started_on) + @request_count.increment(method: env['REQUEST_METHOD'], + path: env['REQUEST_PATH'], + http_status: @status) + [@status, @headers, @response] + end +end diff --git a/src/ui/ui_app.rb b/src/ui/ui_app.rb new file mode 100644 index 0000000..85befd6 --- /dev/null +++ b/src/ui/ui_app.rb @@ -0,0 +1,232 @@ +require 'sinatra' +require 'sinatra/reloader' +require 'json/ext' +require 'haml' +require 'uri' +require 'prometheus/client' +require 'rufus-scheduler' +require 'logger' +require 'faraday' +require 'zipkin-tracer' +require_relative 'helpers' + +# Dependent services +POST_SERVICE_HOST ||= ENV['POST_SERVICE_HOST'] || '127.0.0.1' +POST_SERVICE_PORT ||= ENV['POST_SERVICE_PORT'] || '4567' +COMMENT_SERVICE_HOST ||= ENV['COMMENT_SERVICE_HOST'] || '127.0.0.1' +COMMENT_SERVICE_PORT ||= ENV['COMMENT_SERVICE_PORT'] || '4567' +ZIPKIN_ENABLED ||= ENV['ZIPKIN_ENABLED'] || false +POST_URL ||= "http://#{POST_SERVICE_HOST}:#{POST_SERVICE_PORT}" +COMMENT_URL ||= "http://#{COMMENT_SERVICE_HOST}:#{COMMENT_SERVICE_PORT}" + +# App version and build info +if File.exist?('VERSION') + VERSION ||= File.read('VERSION').strip +else + VERSION ||= "version_missing" +end + +if File.exist?('build_info.txt') + BUILD_INFO = File.readlines('build_info.txt') +else + BUILD_INFO = Array.new(2, "build_info_missing") +end + +@@host_info=ENV['HOSTNAME'] +@@env_info=ENV['ENV'] + +# Zipkin opts +set :zipkin_enabled, ZIPKIN_ENABLED +zipkin_config = { + service_name: 'ui_app', + service_port: 9292, + sample_rate: 1, + sampled_as_boolean: false, + log_tracing: true, + json_api_host: 'http://zipkin:9411/api/v1/spans' + } + +configure do + # https://github.com/openzipkin/zipkin-ruby#sending-traces-on-incoming-requests + http_client = Faraday.new do |faraday| + faraday.use ZipkinTracer::FaradayHandler + faraday.request :url_encoded # form-encode POST params + # faraday.response :logger + faraday.adapter Faraday.default_adapter # make requests with Net::HTTP + end + set :http_client, http_client + set :bind, '0.0.0.0' + set :server, :puma + set :logging, false + set :mylogger, Logger.new(STDOUT) + enable :sessions +end + +if settings.zipkin_enabled? + use ZipkinTracer::RackHandler, zipkin_config +end + +# create and register metrics +prometheus = Prometheus::Client.registry +ui_health_gauge = Prometheus::Client::Gauge.new( + :ui_health, + 'Health status of UI service' +) +ui_health_post_gauge = Prometheus::Client::Gauge.new( + :ui_health_post_availability, + 'Check if Post service is available to UI' +) +ui_health_comment_gauge = Prometheus::Client::Gauge.new( + :ui_health_comment_availability, + 'Check if Comment service is available to UI' +) +prometheus.register(ui_health_gauge) +prometheus.register(ui_health_post_gauge) +prometheus.register(ui_health_comment_gauge) + +# Schedule health check function +scheduler = Rufus::Scheduler.new +scheduler.every '5s' do + check = JSON.parse(http_healthcheck_handler(POST_URL, COMMENT_URL, VERSION)) + set_health_gauge(ui_health_gauge, check['status']) + set_health_gauge(ui_health_post_gauge, check['dependent_services']['post']) + set_health_gauge(ui_health_comment_gauge, check['dependent_services']['comment']) +end + +# before each request +before do + session[:flashes] = [] if session[:flashes].class != Array + env['rack.logger'] = settings.mylogger # set custom logger +end + +# after each request +after do + request_id = env['REQUEST_ID'] || 'null' + logger.info("service=ui | event=request | path=#{env['REQUEST_PATH']} | " \ + "request_id=#{request_id} | " \ + "remote_addr=#{env['REMOTE_ADDR']} | " \ + "method= #{env['REQUEST_METHOD']} | " \ + "response_status=#{response.status}") +end + +# show all posts +get '/' do + @title = 'All posts' + begin + @posts = http_request('get', "#{POST_URL}/posts") + rescue StandardError => e + flash_danger('Can\'t show blog posts, some problems with the post ' \ + 'service. Refresh?') + log_event('error', 'show_all_posts', + "Failed to read from Post service. Reason: #{e.message}") + else + log_event('info', 'show_all_posts', + 'Successfully showed the home page with posts') + end + @flashes = session[:flashes] + session[:flashes] = nil + haml :index +end + +# show a form for creating a new post +get '/new' do + @title = 'New post' + @flashes = session[:flashes] + session[:flashes] = nil + haml :create +end + +# talk to Post service in order to creat a new post +post '/new/?' do + if params['link'] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI] + begin + http_request('post', "#{POST_URL}/add_post", title: params['title'], + link: params['link'], + created_at: Time.now.to_i) + rescue StandardError => e + flash_danger("Can't save your post, some problems with the post service") + log_event('error', 'post_create', + "Failed to create a post. Reason: #{e.message}", params) + else + flash_success('Post successuly published') + log_event('info', 'post_create', 'Successfully created a post', params) + end + redirect '/' + else + flash_danger('Invalid URL') + log_event('warning', 'post_create', 'Invalid URL', params) + redirect back + end +end + +# talk to Post service in order to vote on a post +post '/post/:id/vote/:type' do + begin + http_request('post', "#{POST_URL}/vote", id: params[:id], + type: params[:type]) + rescue StandardError => e + flash_danger('Can\'t vote, some problems with the post service') + log_event('error', 'vote', + "Failed to vote. Reason: #{e.message}", params) + else + log_event('info', 'vote', 'Successful vote', params) + end + redirect back +end + +# show a specific post +get '/post/:id' do + begin + @post = http_request('get', "#{POST_URL}/post/#{params[:id]}") + rescue StandardError => e + log_event('error', 'show_post', + "Counldn't show the post. Reason: #{e.message}", params) + halt 404, 'Not found' + end + + begin + @comments = http_request('get', "#{COMMENT_URL}/#{params[:id]}/comments") + rescue StandardError => e + log_event('error', 'show_post', + "Counldn't show the comments. Reason: #{e.message}", params) + flash_danger("Can't show comments, some problems with the comment service") + else + log_event('info', 'show_post', + 'Successfully showed the post', params) + end + @flashes = session[:flashes] + session[:flashes] = nil + haml :show +end + +# talk to Comment service in order to comment on a post +post '/post/:id/comment' do + begin + http_request('post', "#{COMMENT_URL}/add_comment", + post_id: params[:id], + name: params[:name], + email: params[:email], + created_at: Time.now.to_i, + body: params[:body]) + rescue StandardError => e + log_event('error', 'create_comment', + "Counldn't create a comment. Reason: #{e.message}", params) + flash_danger("Can\'t save the comment, + some problems with the comment service") + else + log_event('info', 'create_comment', + 'Successfully created a new post', params) + + flash_success('Comment successuly published') + end + redirect back +end + +# health check endpoint +get '/healthcheck' do + http_healthcheck_handler(POST_URL, COMMENT_URL, VERSION) +end + +get '/*' do + halt 404, 'Page not found' +end diff --git a/src/ui/views/create.haml b/src/ui/views/create.haml new file mode 100644 index 0000000..172e3da --- /dev/null +++ b/src/ui/views/create.haml @@ -0,0 +1,10 @@ +%h2 Add blog post +%form{ action: '/new', method: 'post', role: 'form'} + .form-group + %label{for: 'title'} Title: + %input{name:'title', placeholder: 'cool title', class: 'form-control', id: 'title'} + .form-group + %label{for: 'link'} Link: + %input{name:'link', placeholder: 'awesome link', class: 'form-control', id: 'link'} + .form-group + %input{class: 'btn btn-primary', type: 'submit', value:'Post it!'} diff --git a/src/ui/views/index.haml b/src/ui/views/index.haml new file mode 100644 index 0000000..96b7e76 --- /dev/null +++ b/src/ui/views/index.haml @@ -0,0 +1,27 @@ +- unless @posts.nil? or @posts.empty? + - @posts.each do |post| + %div{ id: 'postlist'} + %div{class: 'panel'} + %div{class: 'panel-heading'} + %div{class: 'text-center'} + %div{class: 'row'} + .col-sm-1 + %form{:action => "/post/#{post['_id']['$oid']}/vote/1", :method => "post", id: "form-upvote" } + %input{:type => "hidden", :name => "_method", :value => "post"} + %button{type: "submit", class: "btn btn-default btn-sm"} + %span{ class: "glyphicon glyphicon-menu-up" } + %h4{class: 'pull-center'} #{post['votes']} + %form{:action => "/post/#{post['_id']['$oid']}/vote/-1", :method => "post", id: "form-downvote"} + %input{:type => "hidden", :name => "_method", :value=> "post"} + %button{type: "submit", class: "btn btn-default btn-sm"} + %span{ class: "glyphicon glyphicon-menu-down" } + .col-sm-8 + %h3{class: 'pull-left'} + %a{href: "/post/#{post['_id']['$oid']}"} #{post['title']} + .col-sm-3 + %h4{class: 'pull-right'} + %small + %em #{Time.at(post['created_at'].to_i).strftime('%d-%m-%Y')} + %br #{Time.at(post['created_at'].to_i).strftime('%H:%M')} + .panel-footer + %a{href: "#{post['link']}", class: 'btn btn-link'} Go to the link diff --git a/src/ui/views/layout.haml b/src/ui/views/layout.haml new file mode 100644 index 0000000..3072b09 --- /dev/null +++ b/src/ui/views/layout.haml @@ -0,0 +1,35 @@ +!!! 5 +%html(lang="en") + %head + %meta(charset="utf-8") + %meta(http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1") + %meta(name="viewport" content="width=device-width, initial-scale=1.0") + %title="Microservices Reddit :: #{@title}" + %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css', integrity: "sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7", type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' } + %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css', integrity: 'sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r', type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' } + %script{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js', integrity: 'sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS', crossorigin: 'anonymous'} + %body + .navbar.navbar-default.navbar-static-top + .container + %button.navbar-toggle(type="button" data-toggle="collapse" data-target=".navbar-responsive-collapse") + %span.icon-bar + %span.icon-bar + %span.icon-bar + %a.navbar-brand(href="/") Microservices Reddit in #{@@env_info} #{@@host_info} container + .navbar-collapse.collapse.navbar-responsive-collapse + .container + .row + .col-lg-9 + - unless @flashes.nil? or @flashes.empty? + - @flashes.each do |flash| + .alert{class: flash[:type]} + %strong #{flash[:message]} + = yield + .col-lg-3 + .well.sidebar-nav + %h3 Menu + %ul.nav.nav-list + %li + %a(href="/") All posts + %li + %a(href="/new") New post diff --git a/src/ui/views/show.haml b/src/ui/views/show.haml new file mode 100644 index 0000000..07f160f --- /dev/null +++ b/src/ui/views/show.haml @@ -0,0 +1,49 @@ +%div{ id: 'postlist'} + %div{class: 'panel'} + %div{class: 'panel-heading'} + %div{class: 'text-center'} + %div{class: 'row'} + .col-sm-1 + %form{action: "#{@post['_id']['$oid']}/vote/1", method: "post"} + %input{:type => "hidden", :name => "_method", :value => "post"} + %button{type: "submit", class: "btn btn-default btn-sm"} + %span{ class: "glyphicon glyphicon-menu-up" } + %h4{class: 'pull-center'} #{@post['votes']} + %form{:action => "#{@post['_id']['$oid']}/vote/-1", :method => "post"} + %input{:type => "hidden", :name => "_method", :value=> "post"} + %button{type: "submit", class: "btn btn-default btn-sm"} + %span{ class: "glyphicon glyphicon-menu-down" } + .col-sm-8 + %h3{class: 'pull-left'} + %a{href: "#{@post['_id']['$oid']}"} #{@post['title']} + .col-sm-3 + %h4{class: 'pull-right'} + %small + %em #{Time.at(@post['created_at'].to_i).strftime('%d-%m-%Y')} + %br #{Time.at(@post['created_at'].to_i).strftime('%H:%M')} + .panel-footer + %a{href: "#{@post['link']}", class: 'btn btn-link'} Go to the link + +- unless @comments.nil? or @comments.empty? + - @comments.each do |comment| + .row + .col-sm-8 + .panel.panel-default + .panel-heading + %strong #{comment['name']} + - unless comment['email'].empty? + (#{comment['email']}) + %span{class: "text-muted pull-right"} + %em #{Time.at(comment['created_at'].to_i).strftime('%H:%M')} #{Time.at(comment['created_at'].to_i).strftime('%d-%m-%Y')} + .panel-body #{comment['body']} +%form{action: "/post/#{@post['_id']['$oid']}/comment", method: 'post', role: 'form'} + .col-sm-4 + .form-group + %input{name: 'name', type: 'text', class: 'form-control', placeholder: 'name'} + .col-sm-4 + .form-group + %input{name: 'email', type: 'email', class: 'form-control', placeholder: 'email'} + .col-sm-8 + .form-group + %textarea{name: 'body', class: 'form-control', placeholder: 'put a nice comment :)'} + %button{class: 'btn btn-block btn-primary'} Post my comment