Openstack multi-node network architecture:Controller

• 更改編碼

LANG="en_US.utf8"

• 為了讓網卡代號為我們習慣的 ethx 這邊修改 grub

GRUB_DEFAULT=0

GRUB_HIDDEN_TIMEOUT_QUIET=true
GRUB_TIMEOUT=2
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="net.ifnames=0 biosdevname=0"
GRUB_CMDLINE_LINUX=""

• 這邊要設定網卡
  • eth0 為 management network, 使用 192.168.10.0/24 網段
  • eth1 為 external network, 這邊是拿外面 DHCP 使用 192.168.1.0/24 網段
• 要特別注意的是 gateway 不要設到兩個 (含 DHCP) 不然機器會不曉得把封包往哪送

# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

source /etc/network/interfaces.d/*

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto eth0
iface eth0 inet static
	address 192.168.10.1
	netmask 255.255.255.0

auto eth1
iface eth1 inet static
	address 192.168.1.1
	netmask 255.255.255.0
	gateway 192.168.1.253

dns-nameserver 8.8.8.8
dns-search mitaka.openstack

• 因為修改 grub 後網卡名稱改變了，/etc/network/interface 也要一起修改之後才能 reboot，否則直接 reboot 會發現網卡只剩 lo0

# update-grub
# reboot

• 修改 hosts 讓機器可以認到 hostname，127.0.1.1 的那行把它移除

127.0.0.1       localhost
192.168.10.1    ctrl.mitaka.openstack ctrl
192.168.1.1     public.mitaka.openstack public

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

• 確認 hostname:這邊再安裝 ubuntu 時應該就設了，如果有更動記得在此變更，要跟 /etc/hosts match

ctrl

• 更新 pkg

# apt-get update
# apt-get upgrade

• 安裝 ubuntu-cloud-keyring 套件：Ubuntu Cloud Archive 是一個 online software repository，讓我們在 Ubuntu LTS version 上可以下載到最新版本的 Openstack，而 ubuntu-cloud-keyring 是 Ubuntu Cloud Archive 的 GnuPG keys

# apt-get install ubuntu-cloud-keyring

• 更新 pkg 與所有相依套件並重啟系統

# apt-get -y dist-upgrade
# reboot

• 清理系統，移除所有不再被需要的套件

# apt-get -y autoremove --purge

• 為了讓 Openstack 系統中所有 Node 時間同步，這邊需要使用 NTP
• 這邊 NTP server 也可以參考官網使用 chrony 這套

• 安裝 ntp 套件

# apt-get -y install ntp

• 這邊修正預設 pool，改為比較近的 tw server
• 關閉原本聽在所有介面的設定，只聽 192.168.10.1(management interface) 和 192.168.1.1(public interface)
• 限制權限，在 management network 上禁止用戶透過 ntp client(ntpc 與 ntpq 程式) 來修改伺服器時間參數，也禁止 trap 的 remote event logging

...

#pool 0.ubuntu.pool.ntp.org iburst
#pool 1.ubuntu.pool.ntp.org iburst
#pool 2.ubuntu.pool.ntp.org iburst
#pool 3.ubuntu.pool.ntp.org iburst

#pool ntp.ubuntu.com
pool clock.stdtime.gov.tw

interface ignore wildcard
interface listen 192.168.1.1
interface listen 192.168.10.1

...

restrict 192.168.10.0 mask 255.255.255.0 nomodify notrap

...

• 重啟 NTP service

# service ntp restart

• 驗證

# ntpq -p

• 為了讓後續的 network node 和 compute node 可以連上 Internet，這邊要在 controller 上使用 NAT 做 routing

• 首先在 kernel 中 enable packet forwarding，把 net.ipv4.ip_forward=1 註解掉

# Uncomment the next line to enable packet forwarding for IPv4
net.ipv4.ip_forward=1

or

• 用 sed 指令直接代換掉，兩者都可

# sed -i '/^#net\.ipv4\.ip_forward/ s/^.//' /etc/sysctl.conf

• 接著安裝 iptables-persistent pkg

# apt-get -y install iptables-persistent

• 這邊寫 NAT rule 如下，需要特別注意如果 ip 或網卡代號、順序有更動都要做對應的修改
• 因為我之後要讓 Internet 可以直接連上 dashboard，所以在 ctrl 的 iptable 也放行實驗室 IP

*nat
:PREROUTING ACCEPT
:INPUT ACCEPT
:OUTPUT ACCEPT
:POSTROUTING ACCEPT
-A POSTROUTING -o eth1 -j MASQUERADE
COMMIT

*filter
:INPUT DROP
:FORWARD DROP
:OUTPUT ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -p tcp -m state --state ESTABLISHED -j ACCEPT
-A INPUT -p udp -m udp --sport 53 -j ACCEPT
-A INPUT -p udp -m udp --sport 123 -j ACCEPT
-A INPUT -s 140.113.216.224/27 -j ACCEPT
-A INPUT -s 192.168.1.0/24 -j ACCEPT
-A INPUT -s 192.168.10.0/24 -j ACCEPT
-A FORWARD -i eth0 -o eth1 -j ACCEPT
-A FORWARD -i eth1 -o eth0 -j ACCEPT
COMMIT

• 重啟 iptables service

# service netfilter-persistent reload

• bind 是用來架設 DNS Server 的套件，因為有些軟體寫得並不完善，僅靠 hosts file 可能會有意外狀況，所以這邊我們為 Openstack 架設內部使用的 DNS Server

• 安裝 bind 套件

# apt-get -y install bind9

• 設定上我們將不在我們 domain 的 host forward 請 google dns 幫忙查詢
• DNS Server 聽在 192.168.10.1，只幫 management network 做遞迴查詢

options {
	directory "/var/cache/bind";

	forwarders {
	        8.8.8.8;
	};

	dnssec-validation auto;

	auth-nxdomain no;    # conform to RFC1035
	listen-on { 192.168.10.1; };
	allow-recursion { 192.168.10.0/24; };
};

• 接著新增正解和反解 zone

zone "mitaka.openstack." {
	type master;
	file "/etc/bind/mitaka.openstack.zone";
};

zone "168.192.in-addr.arpa." {
	type master;
	file "/etc/bind/168.192.in-addr.arpa.zone";
};

• 正解 zone file 內容如下

$TTL	604800
$ORIGIN	mitaka.openstack.
@	IN	SOA	ctrl.mitaka.openstack root (
			1       ; Serial
			604800  ; Refresh
			86400   ; Retry
			2419200 ; Expire
			604800) ; Negative Cache TTL
;
	        IN	NS	ctrl.mitaka.openstack.
ctrl		IN	A	192.168.10.1
public		IN	A	192.168.1.1
network		IN	A	192.168.10.2
compute-1	IN	A	192.168.10.11
compute-2	IN	A	192.168.10.12
compute-3	IN	A	192.168.10.13

• 反解 zone file 如下

$TTL	604800
$ORIGIN	168.192.in-addr.arpa.
@	IN	SOA	ctrl.mitaka.openstack root (
			1       ; Serial
			604800  ; Refresh
			86400   ; Retry
			2419200 ; Expire
			604800) ; Negative Cache TTL
;
        IN      NS      ctrl.mitaka.openstack.
1.10    IN      PTR     ctrl.mitaka.openstack.
1.1     IN      PTR     public.mitaka.openstack.
2.10    IN      PTR     network.mitaka.openstack.
11.10   IN      PTR     compute-1.mitaka.openstack.
12.10   IN      PTR     compute-2.mitaka.openstack.
13.10   IN      PTR     compute-3.mitaka.openstack.

• 修改 server 查詢的 dns server

# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

source /etc/network/interfaces.d/*

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto eth0
iface eth0 inet dhcpstatic
	address 192.168.10.1
	netmask 255.255.255.0

auto eth1
iface eth1 inet static
	address 192.168.1.1
	netmask 255.255.255.0
	gateway 192.168.1.253

dns-nameserver 192.168.10.1
dns-search mitaka.openstack

• 重啟機器

# reboot

• 驗證

host ctrl.mitaka.openstack

• 在 Openstack 中許多元件都會使用到資料庫，在這邊我們使用 MySQL 取代預設的 sqlite

• 安裝 MySQL 套件

# apt-get -y install mysql-server python-mysqldb

• 更改設定讓 MySQL 聽在 management network，關閉 host-cache，設置編碼為 utf8

[mysqld_safe]
socket          = /var/run/mysqld/mysqld.sock
nice            = 0

[mysqld]
user            = mysql
pid-file        = /var/run/mysqld/mysqld.pid
socket          = /var/run/mysqld/mysqld.sock
port            = 3306
basedir         = /usr
datadir         = /var/lib/mysql
tmpdir          = /tmp
lc-messages-dir = /usr/share/mysql
skip-external-locking
skip-host-cache

bind-address            = 192.168.10.1

key_buffer_size         = 16M
max_allowed_packet      = 16M
thread_stack            = 192K
thread_cache_size       = 8

myisam-recover         = BACKUP

query_cache_limit       = 1M
query_cache_size        = 16M

log_error = /var/log/mysql/error.log

expire_logs_days        = 10
max_binlog_size   = 100M

character_set_server    = utf8
collation_server        = utf8_bin
default_storage_engine  = InnoDB
init_connect            = 'SET NAMES UTF8'

• 接著底下是一個 sql script，裡面包含了我們在 Openstack 終會使用到的資料庫與相對應的帳號權限配置，若參考官方資料可以發現官方是針對各個 project 安裝時會一併建立對應的資料庫資料，這邊則是再安裝前一口氣將所有會用到的資料都建立起來

DROP DATABASE IF EXISTS keystone; CREATE DATABASE keystone;
GRANT ALL PRIVILEGES ON keystone.* TO 'keystoneUser'@'ctrl.mitaka.openstack' IDENTIFIED BY 'keystonePass';
DROP DATABASE IF EXISTS glance; CREATE DATABASE glance;
GRANT ALL PRIVILEGES ON glance.* TO 'glanceUser'@'ctrl.mitaka.openstack' IDENTIFIED BY 'glancePass';
DROP DATABASE IF EXISTS cinder; CREATE DATABASE cinder;
GRANT ALL PRIVILEGES ON cinder.* TO 'cinderUser'@'ctrl.mitaka.openstack' IDENTIFIED BY 'cinderPass';
GRANT ALL PRIVILEGES ON cinder.* TO 'cinderUser'@'compute-%.mitaka.openstack' IDENTIFIED BY 'cinderPass';
DROP DATABASE IF EXISTS nova; CREATE DATABASE nova;
GRANT ALL PRIVILEGES ON nova.* TO 'novaUser'@'ctrl.mitaka.openstack' IDENTIFIED BY 'novaPass';
DROP DATABASE IF EXISTS `nova-api`; CREATE DATABASE `nova-api`;
GRANT ALL PRIVILEGES ON `nova-api`.* TO 'novaUser'@'ctrl.mitaka.openstack' IDENTIFIED BY 'novaPass';
DROP DATABASE IF EXISTS heat; CREATE DATABASE heat;
GRANT ALL PRIVILEGES ON heat.* TO 'heatUser'@'ctrl.mitaka.openstack' IDENTIFIED BY 'heatPass';
DROP DATABASE IF EXISTS neutron; CREATE DATABASE neutron;
GRANT ALL PRIVILEGES ON neutron.* TO 'neutronUser'@'network.mitaka.openstack' IDENTIFIED BY 'neutronPass';

• 初始化資料庫

# service mysql stop
# mysqld --user=mysql --initialize-insecure
# service mysql start
# mysql_secure_installation

• 將剛剛的 sql file 匯入資料庫

# mysql -u root -p < ~/init-database.sql

• 驗證

mysqlshow -u root -p
echo "SELECT user,host FROM user WHERE host LIKE '%.mitaka.openstack';" | mysql -u root -p -t mysql

• RabbitMQ 是一個實作 AMQP(Advanced Message Queuing Protocol) 的軟體，實現訊息佇列。由於 Openstack 是由許多 project 結合而成，這之間需要互相通訊，而 messgae queue 提供了 Async (允許非同步) 和 Reliable (保證傳達) 的訊息傳達交換方式。

• 安裝套件

# apt-get -y install rabbitmq-server

• 修改設定檔，將 node ip 設在 management network

#NODE_NAME=rabbit
NODE_NAME=rabbit@localhost
#NODE_IP_ADDRESS=127.0.0.1
NODE_IP_ADDRESS=192.168.67.94
NODE_PORT=5672

• 底下這個設定是讓 rabbitMQ 聽在 127.0.0.1 而非 0.0.0.0，但實際使用有碰過設定後 rabbitMQ 開不起來的狀況，非必需

[
  {kernel, [
    {inet_dist_use_interface, {127,0,0,1}}
  ]}
].

• 重啟 rabbitMQ
• 在 rabbitMQ 增加 turtle user 密碼為 slowly
• 配置 turtle 權限(全開)

# service rabbitmq-server restart
# rabbitmqctl add_user turtle slowly
# rabbitmqctl set_permissions -p / turtle ".*" ".*" ".*"

• 驗證

# epmd -names
# rabbitmqctl status
# rabbitmqctl list_user_permissions turtle

• Horizon 是 Openstack 中用來提供 web dashboard 的一個 project reference

• 安裝套件

# apt-get -y install openstack-dashboard memcached

• 修改設定讓 Horizon 支援 Openstack v3 認證
  • OPENSTACK_HOST 由 127.0.0.1 修正為 public.mitaka.openstack
  • OPENSTACK_KEYSTONE_URL 中的 v2.0 修正為 v3
  • OPENSTACK_API_VERSIONS 區塊頭尾取消註解，identity 也取消註解
  • OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT 取消註解，False 改為 True

...

OPENSTACK_HOST = "public.mitaka.openstack"
OPENSTACK_KEYSTONE_URL = "http://%s:5000/v3" % OPENSTACK_HOST
OPENSTACK_KEYSTONE_DEFAULT_ROLE = "_member_"

...

OPENSTACK_API_VERSIONS = {
#    "data-processing": 1.1,
    "identity": 3,
#    "volume": 2,
#    "compute": 2,
}

# Set this to True if running on multi-domain model. When this is enabled, it
# will require user to enter the Domain name in addition to username for login.
OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT = True

or

• 用 sed 指令直接代換掉，兩者都可

# sed -i '/^OPENSTACK_HOST =/ s/127\.0\.0\.1/public\.mitaka\.openstack/' /etc/openstack-dashboard/local_settings.py
# sed -i '/^OPENSTACK_KEYSTONE_URL =/ s/v2\.0/v3/' /etc/openstack-dashboard/local_settings.py
# sed -i '/^#OPENSTACK_API_VERSIONS =/,/^#}$/ {/OPENSTACK_API_VERSIONS =/s/^#//; /identity/s/^#//; /}$/s/^#//}' /etc/openstack-dashboard/local_settings.py
# sed -i '/^#OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT/ {s/False$/True/; s/^#//}' /etc/openstack-dashboard/local_settings.py

• 驗證

grep OPENSTACK_HOST /etc/openstack-dashboard/local_settings.py
grep -A 5 'OPENSTACK_API_VERSIONS =' /etc/openstack-dashboard/local_settings.py
grep OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT /etc/openstack-dashboard/local_settings.py

• 更改 apache2 設定，因為我們希望 web dashboard 可以在外部網路直接存取，所以聽在 external interface

Listen 192.168.1.1:80

<IfModule ssl_module>
        Listen 443
</IfModule>

<IfModule mod_gnutls.c>
        Listen 443
</IfModule>

• 重啟相關 service

# service apache2 restart
# service memcached restart

• 驗證

netstat -nlpt | grep ":80"

• Keystone 是 Openstack 中用來提供身分驗證和授權服務的 project reference

• 安裝套件

# apt-get -y install keystone

• 設定檔
  • 主要要注意是 database 這邊我們使用 mysql 而非預設的 sqlite，connect 認證要與前面 sql file 建立的資料相符
  • oslo_messaging_rabbit 使用的帳號密碼也要與前面 rabbitMQ 建立的相符合
  • server 主要聽在 5000 和 35357 port

[DEFAULT]
...

rpc_backend = rabbit

...

[assignment]
#driver = <None>
driver = sql

...

[catalog]
driver = sql

...

[database]
backend = sqlalchemy
#connection = sqlite:////var/lib/keystone/keystone.db
connection = mysql://keystoneUser:keystonePass@ctrl.mitaka.openstack/keystone

...

[eventlet_server]
#public_bind_host = 0.0.0.0
#public_port = 5000
#admin_bind_host = 0.0.0.0
#admin_port = 35357

...

[identity]
default_domain_id = default

driver = sql

...

[memcache]
servers = localhost:11211

...

[oslo_messaging_rabbit]
#rabbit_host = localhost
rabbit_host = ctrl.mitaka.openstack
rabbit_port = 5672
#rabbit_userid = guest
rabbit_userid = turtle
#rabbit_password = guest
rabbit_password = slowly
rabbit_login_method = AMQPLAIN
rabbit_virtual_host = /

...

[revoke]
driver = sql

...

[token]
#provider = uuid
provider = fernet
#driver = sql
driver = memcache

• wsgi(Web Server Gateway Interface) 是 python 中基於 CGI 的延伸，定義了一個標準介面讓網頁程式和 web server 得以溝通，在 openstack auth 中我們會使用到 wsgi，以下為相關設定

• 在 apache 中定義兩個 virtual host 聽在 5000(public/external network) 和 35357(admin/management network)，當有 http request 過來時，背後其實是 /usr/bin/keystone-wsgi-public 和 /usr/bin/keystone-wsgi-admin 這兩隻 python 程式在處理的

<VirtualHost 192.168.1.1:5000>
    WSGIDaemonProcess keystone-public processes=5 threads=1 user=keystone display-name=%{GROUP}
    WSGIProcessGroup keystone-public
    WSGIScriptAlias / /usr/bin/keystone-wsgi-public
    WSGIApplicationGroup %{GLOBAL}
    WSGIPassAuthorization On
    <IfVersion >= 2.4>
      ErrorLogFormat "%{cu}t %M"
    </IfVersion>
    LogLevel info
    ErrorLog /var/log/apache2/keystone-error.log
    CustomLog /var/log/apache2/keystone-access.log combined

    <Directory /usr/bin>
        <IfVersion >= 2.4>
            Require all granted
        </IfVersion>
        <IfVersion < 2.4>
            Order allow,deny
            Allow from all
        </IfVersion>
    </Directory>
</VirtualHost>

<VirtualHost 192.168.10.1:35357>
    WSGIDaemonProcess keystone-admin processes=5 threads=1 user=keystone display-name=%{GROUP}
    WSGIProcessGroup keystone-admin
    WSGIScriptAlias / /usr/bin/keystone-wsgi-admin
    WSGIApplicationGroup %{GLOBAL}
    WSGIPassAuthorization On
    <IfVersion >= 2.4>
      ErrorLogFormat "%{cu}t %M"
    </IfVersion>
    LogLevel info
    ErrorLog /var/log/apache2/keystone-error.log
    CustomLog /var/log/apache2/keystone-access.log combined

    <Directory /usr/bin>
        <IfVersion >= 2.4>
            Require all granted
        </IfVersion>
        <IfVersion < 2.4>
            Order allow,deny
            Allow from all
        </IfVersion>
    </Directory>
</VirtualHost>

• 修改 apache port 設定，多聽 5000 和 35357 port

Listen 192.168.1.1:80
Listen 192.168.1.1:5000
Listen 192.168.10.1:35357

<IfModule ssl_module>
        Listen 443
</IfModule>

<IfModule mod_gnutls.c>
        Listen 443
</IfModule>

• 底下 script 是在裝其它套件時會新增的 keystone 資訊，這邊一樣一口氣把它先處理掉

#!/bin/bash
# Modify these variables as needed
CONTROLLER_HOST=controller.mitaka.openstack
KEYSTONE_API_HOST=public.mitaka.openstack
NEUTRON_HOST=network.mitaka.openstack

export OS_IDENTITY_API_VERSION=3
export OS_AUTH_URL="http://${CONTROLLER_HOST}:35357/v${OS_IDENTITY_API_VERSION}"
ADMIN_PROJECT_NAME=${ADMIN_PROJECT_NAME:-admin}
export OS_PROJECT_NAME=${ADMIN_PROJECT_NAME}
export OS_USERNAME=admin
export OS_PASSWORD=admin_pass
SERVICE_PROJECT_NAME=${SERVICE_PROJECT_NAME:-service}
SERVICE_PASSWORD=${SERVICE_PASSWORD:-service_pass}

KEYSTONE_REGION=${KEYSTONE_REGION:-RegionOne}
DEFAULT_DOMAIN=${DEFAULT_DOMAIN:-default}
ADMIN_ROLE=${ADMIN_ROLE:-admin}

# Services
openstack service create --name glance --description "OpenStack Image service" image
openstack service create --name cinderv2 --description "OpenStack Block Storage" volumev2
openstack service create --name nova --description "OpenStack Compute" compute
openstack service create --name heat --description "Orchestration" orchestration
openstack service create --name heat-cfn --description "Orchestration" cloudformation
openstack service create --name neutron --description "OpenStack Networking" network

# Endpoints
openstack endpoint create --region $KEYSTONE_REGION image public \
        'http://'"$CONTROLLER_HOST"':9292'
openstack endpoint create --region $KEYSTONE_REGION image internal \
        'http://'"$CONTROLLER_HOST"':9292'
openstack endpoint create --region $KEYSTONE_REGION image admin \
        'http://'"$CONTROLLER_HOST"':9292'

openstack endpoint create --region $KEYSTONE_REGION volumev2 public \
        'http://'"$CONTROLLER_HOST"':8776/v2/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION volumev2 internal \
        'http://'"$CONTROLLER_HOST"':8776/v2/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION volumev2 admin \
        'http://'"$CONTROLLER_HOST"':8776/v2/%(tenant_id)s'

openstack endpoint create --region $KEYSTONE_REGION compute public \
        'http://'"$CONTROLLER_HOST"':8774/v2/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION compute internal \
        'http://'"$CONTROLLER_HOST"':8774/v2/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION compute admin \
        'http://'"$CONTROLLER_HOST"':8774/v2/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION orchestration public \
        'http://'"$CONTROLLER_HOST"':8004/v1/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION orchestration internal \
        'http://'"$CONTROLLER_HOST"':8004/v1/%(tenant_id)s'
openstack endpoint create --region $KEYSTONE_REGION orchestration admin \
        'http://'"$CONTROLLER_HOST"':8004/v1/%(tenant_id)s'

openstack endpoint create --region $KEYSTONE_REGION cloudformation public \
        'http://'"$CONTROLLER_HOST"':8000/v1'
openstack endpoint create --region $KEYSTONE_REGION cloudformation internal \
        'http://'"$CONTROLLER_HOST"':8000/v1'
openstack endpoint create --region $KEYSTONE_REGIONcloudformation admin \
        'http://'"$CONTROLLER_HOST"':8000/v1'

openstack endpoint create --region $KEYSTONE_REGION network public \
        'http://'"$NEUTRON_HOST"':9696'
openstack endpoint create --region $KEYSTONE_REGION network internal \
        'http://'"$NEUTRON_HOST"':9696'
openstack endpoint create --region $KEYSTONE_REGION network admin \
        'http://'"$NEUTRON_HOST"':9696'

# Roles
openstack role create _member_
openstack role create heat_stack_owner
openstack role create heat_stack_user

# Projects
openstack project create --domain $DEFAULT_DOMAIN \
        --description "Service Project" $SERVICE_PROJECT_NAME

# Users
openstack user create --domain $DEFAULT_DOMAIN --password $SERVICE_PASSWORD glance
openstack user create --domain $DEFAULT_DOMAIN --password $SERVICE_PASSWORD cinder
openstack user create --domain $DEFAULT_DOMAIN --password $SERVICE_PASSWORD nova
openstack user create --domain $DEFAULT_DOMAIN --password $SERVICE_PASSWORD heat
openstack user create --domain $DEFAULT_DOMAIN --password $SERVICE_PASSWORD neutron

# Add Role
openstack role add --project $ADMIN_PROJECT_NAME --user $ADMIN_PROJECT_NAME $ADMIN_ROLE
openstack role add --project $ADMIN_PROJECT_NAME --user $ADMIN_PROJECT_NAME heat_stack_owner
openstack role add --project $SERVICE_PROJECT_NAME --user glance $ADMIN_ROLE
openstack role add --project $SERVICE_PROJECT_NAME --user cinder $ADMIN_ROLE
openstack role add --project $SERVICE_PROJECT_NAME --user nova $ADMIN_ROLE
openstack role add --project $SERVICE_PROJECT_NAME --user heat $ADMIN_ROLE
openstack role add --project $SERVICE_PROJECT_NAME --user neutron $ADMIN_ROLE

• 因為之後 keystone 會以 wsgi 形式運行，而非 daemon，所以這邊停止服務並取消 keystone service

# service keystone stop
# systemctl disable keystone

• 初始化 database schema

# keystone-manage db_sync
# keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
# keystone-manage bootstrap --bootstrap-admin-url=http://ctrl.mitaka.openstack:35357/v3 --bootstrap-public-url=http://public.mitaka.openstack:5000/v3 --bootstrap-internal-url=http://public.mitaka.openstack:5000/v3 --bootstrap-service-name=keystone --bootstrap-role-name=admin --bootstrap-project-name=admin --bootstrap-username=admin --bootstrap-password=admin_pass

• 啟動 apache2 site (virtual host) 讓 wsgi-keystone 載入
  • a2ensite, a2dissite - enable or disable an apache2 site / virtual host

# service apache2 stop
# a2ensite wsgi-keystone
# service apache2 start

• 驗證 apache2 listening port

netstat -nlpt | grep ":5000"
netstat -nlpt | grep ":35357"

• 之後因為我們使用的 mysql 取代了 keystone 的 sqlite，所以這邊可以把 keystone.db 移除

# rm -f /var/lib/keystone/keystone.db

• 初始化 keystone

# chmod +x ~/init-keystone.sh
# ~/init-keystone.sh

• 驗證 keystone

# openstack --os-auth-url="http://controller.mitaka.openstack:35357/v3" --os-auth-type=v3password --os-project-name=admin --os-project- domain-name=Default --os-username=admin --os-user-domain-name=Default --os-password=admin_pass --os-identity-api-version=3 domain list
# openstack --os-auth-url="http://controller.mitaka.openstack:35357/v3" --os-auth-type=v3password --os-project-name=admin --os-project-domain-name=Default --os-username=admin --os-user-domain-name=Default --os-password=admin_pass --os-identity-api-version=3 project list
# openstack --os-auth-url="http://controller.mitaka.openstack:35357/v3" --os-auth-type=v3password --os-project-name=admin --os-project-domain-name=Default --os-username=admin --os-user-domain-name=Default --os-password=admin_pass --os-identity-api-version=3 role list
# openstack --os-auth-url="http://controller.mitaka.openstack:35357/v3" --os-auth-type=v3password --os-project-name=admin --os-project-domain-name=Default --os-username=admin --os-user-domain-name=Default --os-password=admin_pass --os-identity-api-version=3 user list
# openstack --os-auth-url="http://controller.mitaka.openstack:35357/v3" --os-auth-type=v3password --os-project-name=admin --os-project-domain-name=Default --os-username=admin --os-user-domain-name=Default --os-password=admin_pass --os-identity-api-version=3 service list
# openstack --os-auth-url="http://controller.mitaka.openstack:35357/v3" --os-auth-type=v3password --os-project-name=admin --os-project-domain-name=Default --os-username=admin --os-user-domain-name=Default --os-password=admin_pass --os-identity-api-version=3 endpoint list

• Glance 套件讓我們可以尋找、註冊、取回 virtual machine images。儲存的Image可作為新伺服器部署所需的範本，加快服務上線速度。若是有多臺伺服器需要配置新服務，就不需要額外花費時間單獨設定，也可做為備份時所用。類似 VM 的 import/export reference

• 安裝套件

# apt-get -y install glance

• 修改 glance registry 設定
  • 只聽在 192.168.10.1:9191(management network)
  • 加入 logfile 路徑，方便除錯
  • 資料庫部分使用 mysql

[DEFAULT]
...

owner_is_tenant = true

...

enable_v1_registry = true

...

enable_v2_registry = true

...

#bind_host = 0.0.0.0
bind_host = 192.168.67.94

#bind_port = <None>
bind_port = 9191

...

#log_file = <None>
log_file = /var/log/glance/registry.log

...

[database]
#sqlite_db = /var/lib/glance/glance.sqlite
backend = sqlalchemy
#connection = <None>
connection = mysql://glanceUser:glancePass@ctrl.mitaka.openstack/glance

...

[keystone_authtoken]
#auth_uri = <None>
#identity_uri = <None>
#admin_user = <None>
#admin_password = <None>
#admin_tenant_name = admin

[oslo_messaging_rabbit]
#rabbit_host = localhost
#rabbit_port = 5672
#rabbit_use_ssl = false
#rabbit_userid = guest
#rabbit_password = guest
#rabbit_virtual_host = /

• 修改 glance api 設定，與 glance register 雷同
  • 只聽在 192.168.10.1(management network)
  • api 的 bind port 為 9292
  • registry_host 為 management network 的 controller node
  • 加入 logfile path
  • 資料庫使用 MySQL
  • 指定 filesystem_store_datadir
  • 設定 filesystem_store_file_perm 檔案權限
  • keystone auth_type 用 v3，帳號密碼要符合上面 keystone shell script 建立時的帳密

[DEFAULT]
owner_is_tenant = true
enable_v1_api = true
enable_v2_api = true

#bind_host = 0.0.0.0
bind_host = 192.168.10.1

#bind_port = <None>
bind_port = 9292

#registry_host = 0.0.0.0
registry_host = ctrl.mitaka.openstack

registry_port = 9191

#auth_strategy = noauth
auth_strategy = keystone

registry_client_protocol = http

#log_file = <None>
log_file = /var/log/glance/api.log

[database]
#sqlite_db = /var/lib/glance/glance.sqlite
backend = sqlalchemy
#connection = <None>
connection = mysql://glanceUser:glancePass@ctrl.mitaka.openstack/glance

[glance_store]
stores = file,http

default_store = file

#filesystem_store_datadir = <None>
filesystem_store_datadir = /var/lib/glance/images/

#filesystem_store_file_perm = 0
filesystem_store_file_perm = 644

[image_format]
disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar

[keystone_authtoken]
#auth_uri = <None>
#auth_version = <None>
#memcached_servers = <None>

#auth_type = <None>
auth_type = v3password
auth_url = http://public.mitaka.openstack:5000/v3
project_name = service
project_domain_name = Default
username = glance
user_domain_name = Default
password = service_pass

[oslo_concurrency]
#lock_path = <None>
lock_path = /var/lock/glance

[oslo_messaging_rabbit]
#rabbit_host = localhost
#rabbit_port = 5672
#rabbit_use_ssl = false
#rabbit_userid = guest
#rabbit_password = guest
#rabbit_virtual_host = /

[paste_deploy]
#flavor =  <None>
flavor = keystone

• 初始化 database schema

# glance-manage db_sync

• 重啟 glance service

# service glance-registry restart
# service glance-api restart

• 驗證(空的沒有噴 error 即可)

# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=glance --os-user-domain-name=Default --os-password=service_pass --os-image-api-version=1 image list
# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=glance --os-user-domain-name=Default --os-password=service_pass --os-image-api-version=2 image list

• reference

• 安裝套件

# apt-get -y install cinder-api cinder-scheduler

• 修改設定檔

[DEFAULT]
rootwrap_config = /etc/cinder/rootwrap.conf
api_paste_confg = /etc/cinder/api-paste.ini
#iscsi_helper = tgtadm
#volume_name_template = volume-%s
#volume_group = cinder-volumes
#verbose = True
#auth_strategy = keystone
state_path = /var/lib/cinder
#lock_path = /var/lock/cinder
#volumes_dir = /var/lib/cinder/volumes

enable_v1_api = false
enable_v2_api = true

rpc_backend = rabbit
osapi_volume_listen = 192.168.10.1
osapi_volume_listen_port = 8776

[oslo_concurrency]
lock_path = /var/lock/cinder

[oslo_messaging_rabbit]
rabbit_host = ctrl.mitaka.openstack
rabbit_port = 5672
rabbit_userid = turtle
rabbit_password = slowly
rabbit_login_method = AMQPLAIN
rabbit_virtual_host = /

[database]
backend = sqlalchemy
connection = mysql://cinderUser:cinderPass@ctrl.mitaka.openstack/cinder

[keystone_authtoken]
auth_type = v3password
auth_url = http://public.mitaka.openstack:5000/v3
project_name = service
project_domain_name = Default
username = cinder
user_domain_name = Default
password = service_pass

[keymgr]
encryption_auth_url = http://public.mitaka.openstack:5000/v3

• 初始化 database schema

# cinder-manage db sync

• 重啟 cinder service

# service cinder-scheduler restart
# service cinder-api restart

• 清除 sqlite db

# rm -f /var/lib/cinder/cinder.sqlite

• 驗證

# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=cinder --os-user-domain-name=Default --os-password=service_pass --os-volume-api-version=2 volume list
# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=cinder --os-user-domain-name=Default --os-password=service_pass --os-volume-api-version=2 backup list
# cinder --os-auth-url="http://public.mitaka.openstack:5000/v2.0" --os-tenant-name=service --os-username=cinder --os-password=service_pass --os-volume-api-version=2 list
# cinder --os-auth-url="http://public.mitaka.openstack:5000/v2.0" --os-tenant-name=service --os-username=cinder --os-password=service_pass --os-volume-api-version=2 service-list
# cinder --os-auth-url="http://public.mitaka.openstack:5000/v2.0" --os-tenant-name=service --os-username=cinder --os-password=service_pass --os-volume-api-version=2 availability-zone-list

• Nova 負責管理OpenStack 環境中 compute instances 的生命週期，按需求啟動、調度、關閉虛擬機 referance

• 安裝套件

# apt-get -y install nova-api nova-cert nova-consoleauth nova-scheduler nova-conductor nova-spiceproxy

• 修改設定檔

[DEFAULT]
#dhcpbridge_flagfile=/etc/nova/nova.conf
#dhcpbridge=/usr/bin/nova-dhcpbridge
logdir = /var/log/nova
state_path = /var/lib/nova
#lock_path=/var/lock/nova
#force_dhcp_release=True
force_dhcp_release = true
#libvirt_use_virtio_for_bridges=True
#verbose=True
#ec2_private_dns_show_ip=True
#api_paste_config=/etc/nova/api-paste.ini
#enabled_apis=ec2,osapi_compute,metadata
enabled_apis = osapi_compute,metadata


network_manager = nova.network.manager.VlanManager

osapi_compute_listen = 192.168.10.1
osapi_compute_listen_port = 8774
metadata_listen = 192.168.10.1
metadata_listen_port = 8775

use_neutron = true
rpc_backend = rabbit

[oslo_messaging_rabbit]
rabbit_host = ctrl.mitaka.openstack
rabbit_port = 5672
rabbit_userid = turtle
rabbit_password = slowly
rabbit_login_method = AMQPLAIN
rabbit_virtual_host = /

[oslo_concurrency]
lock_path = /var/lock/nova

[spice]
agent_enabled = true
enabled = false
html5proxy_host = 192.168.1.1
html5proxy_port = 6082

[neutron]
service_metadata_proxy = true
metadata_proxy_shared_secret = helloOpenStack
url = http://network.mitaka.openstack:9696
auth_type = v3password
auth_url = http://public.mitaka.openstack:5000/v3
project_name = service
project_domain_name = Default
username = neutron
user_domain_name = Default
password = service_pass

[glance]
api_servers = http://ctrl.mitaka.openstack:9292

[api_database]
connection = mysql://novaUser:novaPass@ctrl.mitaka.openstack/nova-api

[database]
backend = sqlalchemy
connection = mysql://novaUser:novaPass@ctrl.mitaka.openstack/nova

[keystone_authtoken]
auth_type = v3password
auth_url = http://public.mitaka.openstack:5000/v3
project_name = service
project_domain_name = Default
username = nova
user_domain_name = Default
password = service_pass

• 初始化 database schema

# nova-manage db sync
# nova-manage api_db sync

• 重啟 nova service

# service nova-cert restart
# service nova-conductor restart
# service nova-consoleauth restart
# service nova-spiceproxy restart
# service nova-scheduler restart
# service nova-api restart

• 清除 sqlite db

# rm -f /var/lib/nova/nova.sqlite

• 驗證

# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=nova --os-user-domain-name=Default --os-password=service_pass --os-compute-api-version=2 flavor list
# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=nova --os-user-domain-name=Default --os-password=service_pass --os-compute-api-version=2 availability zone list
# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=nova --os-user-domain-name=Default --os-password=service_pass --os-compute-api-version=2 host list
# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=nova --os-user-domain-name=Default --os-password=service_pass --os-compute-api-version=2 compute service list
# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=nova --os-user-domain-name=Default --os-password=service_pass --os-compute-api-version=2 hypervisor list

• reference

• 安裝套件

# apt-get -y install heat-api heat-api-cfn heat-api-cloudwatch heat-engine

• 修改設定檔

[DEFAULT]
log_dir = /var/log/heat

rpc_backend = rabbit

[database]
backend = sqlalchemy

#connection = <None>
connection = mysql://heatUser:heatPass@controller.mitaka.openstack/heat

[keystone_authtoken]
#auth_uri = <None>
#identity_uri = <None>
#admin_user = <None>
#admin_password = <None>
#admin_tenant_name = admin
#auth_type = <None>
auth_type = v3password
auth_url = http://public.mitaka.openstack:5000/v3
project_name = service
project_domain_name = Default
username = heat
user_domain_name = Default
password = service_pass

[oslo_messaging_rabbit]
#rabbit_host = localhost
rabbit_host = controller.mitaka.openstack

rabbit_port = 5672

#rabbit_userid = guest
rabbit_userid = turtle

#rabbit_password = guest
rabbit_password = slowly

rabbit_login_method = AMQPLAIN

rabbit_virtual_host = /

[heat_api]
bind_host = 192.168.10.1
bind_port = 8004

[heat_api_cfn]
bind_host = 192.168.10.1
bind_port = 8000

[heat_api_cloudwatch]
bind_host = 192.168.10.1
bind_port = 8003

• 初始化 database schema

# heat-manage db_sync

• 重啟 heat service

# service heat-engine restart
# service heat-api restart
# service heat-api-cfn restart
# service heat-api-cloudwatch restart

• 清除 sqlite db

# rm -f /var/lib/heat/heat.sqlite

• 驗證

# openstack --os-auth-url="http://public.mitaka.openstack:5000/v3" --os-auth-type=v3password --os-project-name=service --os-project-domain-name=Default --os-username=heat --os-user-domain-name=Default --os-password=service_pass --os-orchestration-api-version=1 stack list
# heat --os-auth-url="http://public.mitaka.openstack:5000/v2.0" --os-tenant-name=service --os-username=heat --os-password=service_pass --heat-api-version=1 service-list

• APT upgrade 和 dist-upgrade 的差別
• mysql 解决数据库连接变慢