diff --git a/devops/Dockerfile b/devops/Dockerfile new file mode 100644 index 0000000..942ffcc --- /dev/null +++ b/devops/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get install -y python3.9 && apt-get -y install python3-pip +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 && update-alternatives --set python /usr/bin/python3.9 +RUN apt-get install -y curl + +## allows the container to continue running regardless of the PostgreSQL service inside the container +ENTRYPOINT ["tail", "-f", "/dev/null"] +## downside: need to manually start PostgreSQL in the container once the container is running diff --git a/devops/README.md b/devops/README.md new file mode 100644 index 0000000..2c1c90f --- /dev/null +++ b/devops/README.md @@ -0,0 +1,21 @@ +Installing tools +---------------- +1. Ansible +2. Capistrano + +Installing server tools +----------------------- +- Modify the server IP where tagbase-server will be deployed. +- Open the `devops/inventory` file and modify the IPs specified under `prod_server` and `dev_servers` tags. +- Run the following ansible command +``` +ansible-playbook -i ./inventory tagserver_playbook.yaml +``` + +Deploying tagbase-server +------------------------ +- Run the following capistrano to deploy into the staging server +``` +cap staging deploy +``` + diff --git a/devops/ansible.cfg b/devops/ansible.cfg new file mode 100644 index 0000000..f8783ec --- /dev/null +++ b/devops/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +auto_legacy = /usr/local/bin/python3 + + diff --git a/devops/capistrano/Capfile b/devops/capistrano/Capfile new file mode 100644 index 0000000..42dd3a4 --- /dev/null +++ b/devops/capistrano/Capfile @@ -0,0 +1,38 @@ +# Load DSL and set up stages +require "capistrano/setup" + +# Include default deployment tasks +require "capistrano/deploy" + +# Load the SCM plugin appropriate to your project: +# +# require "capistrano/scm/hg" +# install_plugin Capistrano::SCM::Hg +# or +# require "capistrano/scm/svn" +# install_plugin Capistrano::SCM::Svn +# or +require "capistrano/scm/git" +install_plugin Capistrano::SCM::Git + +# Include tasks from other gems included in your Gemfile +# +# For documentation on these, see for example: +# +# https://github.com/capistrano/rvm +# https://github.com/capistrano/rbenv +# https://github.com/capistrano/chruby +# https://github.com/capistrano/bundler +# https://github.com/capistrano/rails +# https://github.com/capistrano/passenger +# +# require "capistrano/rvm" +# require "capistrano/rbenv" +# require "capistrano/chruby" +# require "capistrano/bundler" +# require "capistrano/rails/assets" +# require "capistrano/rails/migrations" +# require "capistrano/passenger" + +# Load custom tasks from `lib/capistrano/tasks` if you have any defined +Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } diff --git a/devops/capistrano/Gemfile b/devops/capistrano/Gemfile new file mode 100644 index 0000000..d9cadc2 --- /dev/null +++ b/devops/capistrano/Gemfile @@ -0,0 +1,5 @@ +source "https://rubygems.org" +group :development do + gem "capistrano", "~> 3.10", require: false + gem "capistrano-rails", "~> 1.3", require: false +end diff --git a/devops/capistrano/README-capistrano.md b/devops/capistrano/README-capistrano.md new file mode 100644 index 0000000..fc792ac --- /dev/null +++ b/devops/capistrano/README-capistrano.md @@ -0,0 +1,5 @@ +Installation: +- Installation instructions https://www.digitalocean.com/community/tutorials/how-to-use-capistrano-to-automate-deployments-getting-started +- To install capistrano run: `gem install capistrano` +- Then install capistrano app dependencies: `bundle install` + diff --git a/devops/capistrano/config/deploy.rb b/devops/capistrano/config/deploy.rb new file mode 100644 index 0000000..b793066 --- /dev/null +++ b/devops/capistrano/config/deploy.rb @@ -0,0 +1,80 @@ +# config valid for current version and patch releases of Capistrano +lock "~> 3.17.1" + +set :application, "tagbase-server" +set :repo_url, "https://github.com/tagbase/tagbase-server.git" # git@github.com:tagbase/tagbase-server.git" + +# Default branch is :master +# ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp + +# Default deploy_to directory is /var/www/my_app_name +set :deploy_to, "/home/ubuntu/tagbase-server" + +# Default value for :format is :airbrussh. +# set :format, :airbrussh + +# You can configure the Airbrussh format using :format_options. +# These are the defaults. +# set :format_options, command_output: true, log_file: "log/capistrano.log", color: :auto, truncate: :auto + +# Default value for :pty is false +set :pty, true + +# Default value for :linked_files is [] +# append :linked_files, "config/database.yml", 'config/master.key' + +# Default value for linked_dirs is [] +# append :linked_dirs, "log", "tmp/pids", "tmp/cache", "tmp/sockets", "tmp/webpacker", "public/system", "vendor", "storage" + +# Default value for default_env is {} +# set :default_env, { path: "/opt/ruby/bin:$PATH" } +set :default_env, { 'COMPOSE_PROJECT_NAME': "tagbase" } + +# Default value for local_user is ENV['USER'] +# set :local_user, -> { `git config user.name`.chomp } + +# Default value for keep_releases is 5 +# set :keep_releases, 5 +# +set :stages, ["staging", "production"] +set :default_stage, "staging" + + +# Uncomment the following to require manually verifying the host key before first deploy. +# set :ssh_options, verify_host_key: :secure + +namespace :deploy do + + desc 'Create certificates for NGINX' + task :copy_env_app do + on roles(:app) do + execute '(cp /home/ubuntu/tagbase-server/.env /home/ubuntu/tagbase-server/current;)' + execute '(mkdir -p /home/ubuntu/tagbase-server/current/services/nginx/ssl;)' + execute '(cd /home/ubuntu/tagbase-server/current/services/nginx/ssl; openssl req -x509 -nodes -newkey rsa:2048 -keyout key.pem -out cert.pem -sha256 -days 365 -subj "/C=GB/ST=London/L=London/O=Alros/OU=IT Department/CN=localhost")' + end + end + + desc 'Rebuild dependencies' + task :rebuild_deps do + on roles(:app) do + # Your restart mechanism here, for example: + execute '(cd /home/ubuntu/tagbase-server/current; docker-compose build --build-arg NGINX_PASS="tagbase" --build-arg NGINX_USER="tagbase" --build-arg PGBOUNCER_PORT=“6432” --build-arg POSTGRES_PASSWORD="tagbase" --build-arg POSTGRES_PORT="5432")' + + #execute "(cd /home/ubuntu/tagbase-server/current; sudo docker-compose down; sudo docker-compose -p'tagbase' up -d)" + execute "(cd /home/ubuntu/tagbase-server/current; sudo docker-compose down; sudo docker-compose up -d)" + end + end + + task :restart_app do + on roles(:app) do + # Your restart mechanism here, for example: + execute "(cd /home/ubuntu/tagbase-server/current; sudo docker-compose down)" + execute "(cd /home/ubuntu/tagbase-server/current; sudo docker-compose up -d)" + end + end + after :deploy, 'deploy:copy_env_app' + after :deploy, 'deploy:rebuild_deps' + after :deploy, 'deploy:restart_app' +end + + diff --git a/devops/capistrano/config/deploy/production.rb b/devops/capistrano/config/deploy/production.rb new file mode 100644 index 0000000..8fb6ec7 --- /dev/null +++ b/devops/capistrano/config/deploy/production.rb @@ -0,0 +1,63 @@ +set :branch, "main" + +# server-based syntax +# ====================== +# Defines a single server with a list of roles and multiple properties. +# You can define all roles on a single server, or split them: + +# server "example.com", user: "deploy", roles: %w{app db web}, my_property: :my_value +# server "example.com", user: "deploy", roles: %w{app web}, other_property: :other_value +# server "db.example.com", user: "deploy", roles: %w{db} + + + +# role-based syntax +# ================== + +# Defines a role with one or multiple servers. The primary server in each +# group is considered to be the first unless any hosts have the primary +# property set. Specify the username and a domain or IP for the server. +# Don't use `:all`, it's a meta role. + +# role :app, %w{deploy@example.com}, my_property: :my_value +# role :web, %w{user1@primary.com user2@additional.com}, other_property: :other_value +# role :db, %w{deploy@example.com} + + + +# Configuration +# ============= +# You can set any configuration variable like in config/deploy.rb +# These variables are then only loaded and set in this stage. +# For available Capistrano configuration variables see the documentation page. +# http://capistranorb.com/documentation/getting-started/configuration/ +# Feel free to add new variables to customise your setup. + + + +# Custom SSH Options +# ================== +# You may pass any option but keep in mind that net/ssh understands a +# limited set of options, consult the Net::SSH documentation. +# http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start +# +# Global options +# -------------- +# set :ssh_options, { +# keys: %w(/home/user_name/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(password) +# } +# +# The server-based syntax can be used to override options: +# ------------------------------------ +# server "example.com", +# user: "user_name", +# roles: %w{web app}, +# ssh_options: { +# user: "user_name", # overrides user setting above +# keys: %w(/home/user_name/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey password) +# # password: "please use keys" +# } diff --git a/devops/capistrano/config/deploy/staging.rb b/devops/capistrano/config/deploy/staging.rb new file mode 100644 index 0000000..92e5d1c --- /dev/null +++ b/devops/capistrano/config/deploy/staging.rb @@ -0,0 +1,67 @@ +set :branch, "main" #149_configuration_tool" +set :server_ip, "18.118.1.177"# 3.15.0.82" + +# server-based syntax +# ====================== +# Defines a single server with a list of roles and multiple properties. +# You can define all roles on a single server, or split them: + +# server "example.com", user: "deploy", roles: %w{app db web}, my_property: :my_value +#server "3.145.103.174", user: "ubuntu", roles: %w{app} +server fetch(:server_ip), user: "ubuntu", roles: %w{app} +# server "example.com", user: "deploy", roles: %w{app web}, other_property: :other_value +# server "db.example.com", user: "deploy", roles: %w{db} + + + +# role-based syntax +# ================== + +# Defines a role with one or multiple servers. The primary server in each +# group is considered to be the first unless any hosts have the primary +# property set. Specify the username and a domain or IP for the server. +# Don't use `:all`, it's a meta role. + +#role :app, %w{ubuntu@3.145.103.174} +role :app, "ubuntu@#{fetch(:server_ip)}" +# role :web, %w{user1@primary.com user2@additional.com}, other_property: :other_value +# role :db, %w{deploy@example.com} + + + +# Configuration +# ============= +# You can set any configuration variable like in config/deploy.rb +# These variables are then only loaded and set in this stage. +# For available Capistrano configuration variables see the documentation page. +# http://capistranorb.com/documentation/getting-started/configuration/ +# Feel free to add new variables to customise your setup. + + + +# Custom SSH Options +# ================== +# You may pass any option but keep in mind that net/ssh understands a +# limited set of options, consult the Net::SSH documentation. +# http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start +# +# Global options +# -------------- +set :ssh_options, { + keys: %w(/Users/renatomarroquin/Documents/workspace/tagbase/tagbase-server/devops/keys/devserver-keypair.pem), + forward_agent: false, + #auth_methods: %w(password) +} +# +# The server-based syntax can be used to override options: +# ------------------------------------ +# server "example.com", +# user: "user_name", +# roles: %w{web app}, +# ssh_options: { +# user: "user_name", # overrides user setting above +# keys: %w(/home/user_name/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey password) +# # password: "please use keys" +# } diff --git a/devops/configure_servers.py b/devops/configure_servers.py new file mode 100644 index 0000000..8da4348 --- /dev/null +++ b/devops/configure_servers.py @@ -0,0 +1,55 @@ +import os +import subprocess +import argparse + + +def handle_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--tagbase-server-home', type=str) + parser.add_argument('--installation-user', type=str) + parser.add_argument('--prod-servers', type=str, nargs='*') + parser.add_argument('--dev-servers', type=str, nargs='*') + parser.add_argument('--dev-servers-sshkey-path', type=str) + parser.add_argument('--prod-servers-sshkey-path', type=str) + return vars(parser.parse_args()) + + +def log_tb(msg): + print("[TAGBASE] {}".format(msg)) + + +def create_ansible_servers_file(inventory_file_path, prod_servers, dev_servers, installation_user, dev_servers_sshkey_path, prod_servers_sshkey_path): + with open(inventory_file_path, 'w') as inv_file: + if prod_servers != None: + inv_file.write("[prod_servers]\n") + for prod_server in prod_servers: + inv_file.write("{}\n".format(prod_server)) + inv_file.write("[prod_servers:vars]\n") + inv_file.write("ansible_user={}\n".format(installation_user)) + inv_file.write("ansible_ssh_private_key_file={}\n".format(prod_servers_sshkey_path)) + + if dev_servers != None: + inv_file.write("[dev_servers]\n") + for dev_server in dev_servers: + inv_file.write("{}\n".format(dev_server)) + inv_file.write("[dev_servers:vars]\n") + inv_file.write("ansible_user={}\n".format(installation_user)) + inv_file.write("ansible_ssh_private_key_file={}\n".format(dev_servers_sshkey_path)) + + log_tb("Completed writing {}".format(inventory_file_path)) + + +def run_ansible_inventory_file(tagserver_home, inventory_file_path): + playbook_file_path = "{}/tagserver_playbook.yaml".format(tagserver_home) + ansible_cmd = "ansible-playbook -i {} {} -v".format(inventory_file_path, playbook_file_path) + subprocess.run(ansible_cmd, shell=True) + log_tb("Completed configuring servers") + + +if __name__ == "__main__": + args = handle_args() + tagbase_server_home = args['tagbase_server_home'] + tagbase_server_devops_home = '{}/{}'.format(tagbase_server_home, 'tagbase-server/devops') + inventory_file_path = '{}/inventory_2'.format(tagbase_server_devops_home) + create_ansible_servers_file(inventory_file_path, args['prod_servers'], args['dev_servers'], args['installation_user'], args['dev_servers_sshkey_path'], args['prod_servers_sshkey_path']) + #run_ansible_inventory_file(tagserver_home, inventory_file_path) diff --git a/devops/inventory b/devops/inventory new file mode 100644 index 0000000..955b713 --- /dev/null +++ b/devops/inventory @@ -0,0 +1,13 @@ +#[prod_servers] +#one.example.com +#ansible_env.TAGBASE_DEV_SERVER + +[dev_servers] +18.118.107.90 + +#one.example.com + +[dev_servers:vars] +ansible_user=ubuntu +ansible_ssh_private_key_file=/Users/renatomarroquin/Documents/workspace/tagbase/devserver-keypair.pem + diff --git a/devops/tagserver_playbook.yaml b/devops/tagserver_playbook.yaml new file mode 100644 index 0000000..1cd4e40 --- /dev/null +++ b/devops/tagserver_playbook.yaml @@ -0,0 +1,75 @@ +--- +- hosts: dev_servers + gather_facts: yes + become: true + + tasks: + - name: Update and upgrade apt packages + become: true + apt: + upgrade: yes + update_cache: yes + cache_valid_time: 86400 # one day validity + + - name: Install apt-transport-https + ansible.builtin.apt: + name: + - apt-transport-https + - ca-certificates + - lsb-release + - gnupg + state: latest + update_cache: true + + - name: Add signing key + ansible.builtin.apt_key: + url: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg" + state: present + + - name: Add repository into sources list + ansible.builtin.apt_repository: + repo: "deb [arch={{ ansible_architecture }}] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable" + state: present + filename: docker + + - name: Install Docker + ansible.builtin.apt: + name: + - docker + - docker.io + - docker-compose + - docker-registry + state: latest + update_cache: true + + - name: Install git - curl - ssh client + apt: + name: "{{ item }}" + state: installed + state: "{{ item.state | default('present') }}" + with_items: + - git + - curl + - openssh-client + - net-tools + - postgresql-client + + - name: Create dir for project + command: mkdir -p /home/ubuntu/tagbase-server/ + + - name: Change ownership + command: chown ubuntu:ubuntu -R /home/ubuntu/tagbase-server/ + + - name: Setting up tagbase-server env file + become: false + copy: + content: | + PGADMIN_DEFAULT_EMAIL=tagtuna@gmail.com + PGADMIN_DEFAULT_PASSWORD=tagbase + POSTGRES_PASSWORD=tagbase + POSTGRES_PORT=5432 + SLACK_BOT_CHANNEL=tagbase-server + SLACK_BOT_TOKEN=XYZXYZ + NGINX_USER=tagbase + NGINX_PASS=tagbase + dest: /home/ubuntu/tagbase-server/.env diff --git a/docker-compose.yml b/docker-compose.yml index cdc17f3..877e853 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: volumes: - ./dbbackups:/backups restart: on-failure + command: tail -f /dev/null docker-cron: build: context: ./services/docker-cron @@ -39,6 +40,7 @@ services: networks: - internal-network restart: unless-stopped + command: tail -f /dev/null fswatch: build: context: ./services/fswatch @@ -76,6 +78,7 @@ services: - ./services/nginx/proxy/:/usr/share/nginx/html/:ro - ./services/nginx/ssl/cert.pem:/etc/nginx/certs/cert.pem - ./services/nginx/ssl/key.pem:/etc/nginx/certs/key.pem + command: tail -f /dev/null pgadmin4: depends_on: postgis: @@ -122,6 +125,7 @@ services: # restart: unless-stopped # volumes: # - ./postgres-data:/var/lib/postgresql/data + command: tail -f /dev/null postgis: environment: - ALLOW_IP_RANGE=0.0.0.0/0 @@ -153,6 +157,7 @@ services: - ./dbbackups:/backups - ./postgis-data:/var/lib/postgresql - ./services/postgis/tagbase_schema.sql:/docker-entrypoint-initdb.d/tagbase_schema.sql + #command: tail -f /dev/null - ./services/postgis/lockfiles:/opt/data/ slack_docker: environment: @@ -164,6 +169,7 @@ services: "docker_compose_diagram.icon": "docker" volumes: - /var/run/docker.sock:/var/run/docker.sock + command: tail -f /dev/null tagbase_server: build: context: ./tagbase_server @@ -190,5 +196,6 @@ services: restart: unless-stopped volumes: - ./logs:/usr/src/app/logs + command: tail -f /dev/null networks: internal-network: \ No newline at end of file diff --git a/services/docker-cron/Dockerfile b/services/docker-cron/Dockerfile index cbeb5d9..bc8d3e9 100644 --- a/services/docker-cron/Dockerfile +++ b/services/docker-cron/Dockerfile @@ -34,4 +34,5 @@ RUN chmod 777 -R /usr/src/app USER tagbase -ENTRYPOINT ["./entrypoint.sh"] +#ENTRYPOINT ["./entrypoint.sh"] +RUN cron diff --git a/tagbase_server/Dockerfile b/tagbase_server/Dockerfile index ba47906..a6572eb 100644 --- a/tagbase_server/Dockerfile +++ b/tagbase_server/Dockerfile @@ -9,6 +9,7 @@ COPY requirements.txt /usr/src/app/ RUN apt update && \ apt -y upgrade && \ apt -y install bash gcc musl-dev tzdata && \ + apt -y install libpq-dev python3-dev && \ python3 -m pip install -r requirements.txt --no-cache-dir RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime