ovs-cni
ovs-cni copied to clipboard
Published
20 hours ago •
John-Lin
John-Lin
Kubernetes NodePort 無法存取Pod
想請教一些問題
我使用kubeadm去建立kubernetes Cluser並使用ovs-cni plugin，依照README流程
修改 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
master加入Environment="KUBELET_EXTRA_ARGS=--node-ip=10.0.0.230"
並在node1加入Environment="KUBELET_EXTRA_ARGS=--node-ip=10.0.0.231"
並使用kubeadm init --apiserver-advertise-address=10.0.0.230
將/etc/cni/net.d依照下面的config設定
在建立網路之後，我使用NodePort 去存取Pod，但是在Cluser中只能存取Pod所在的Node
Kubernetes version:
Client Version: version.Info{Major:"1", Minor:"10", GitVersion:"v1.10.3", GitCommit:"2bba0127d85d5a46ab4b778548be28623b32d0b0", GitTreeState:"clean", BuildDate:"2018-05-21T09:17:39Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"10", GitVersion:"v1.10.3", GitCommit:"2bba0127d85d5a46ab4b778548be28623b32d0b0", GitTreeState:"clean", BuildDate:"2018-05-21T09:05:37Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
OS (e.g. from /etc/os-release):
NAME="Ubuntu"
VERSION="16.04.4 LTS (Xenial Xerus)"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 16.04.4 LTS"
VERSION_ID="16.04"
HOME_URL="http://www.ubuntu.com/"
SUPPORT_URL="http://help.ubuntu.com/"
BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
VERSION_CODENAME=xenial
UBUNTU_CODENAME=xenial
Kernel:
Linux node1 4.4.0-127-generic #153-Ubuntu SMP Sat May 19 10:58:46 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
master cni config:
{
"name":"mynet",
"cniVersion":"0.3.1",
"type":"ovs",
"ovsBridge":"br0",
"vtepIPs":[
"10.0.0.230",
"10.0.0.231"
],
"isDefaultGateway": true,
"ipMasq": true,
"ipam":{
"type":"host-local",
"subnet":"10.244.0.0/16",
"rangeStart":"10.244.1.10",
"rangeEnd":"10.244.1.150",
"routes":[
{
"dst":"0.0.0.0/0"
}
],
"gateway":"10.244.1.1"
}
}
node1 cni config:
{
"name":"mynet",
"cniVersion":"0.3.1",
"type":"ovs",
"ovsBridge":"br0",
"vtepIPs":[
"10.0.0.230",
"10.0.0.231"
],
"isDefaultGateway": true,
"ipMasq": true,
"ipam":{
"type":"host-local",
"subnet":"10.244.0.0/16",
"rangeStart":"10.244.2.10",
"rangeEnd":"10.244.2.150",
"routes":[
{
"dst":"0.0.0.0/0"
}
],
"gateway":"10.244.2.1"
}
}
@John-Lin 我使用kubectl指令建立出nginx deployment以及expose nginx service。
root@host1:/home/vagrant#kubectl run nginx --image nginx
root@host1:/home/vagrant#kubectl expose deploy nginx --type NodePort --port 80
root@host1:/home/vagrant# kubectl get pod,svc
NAME READY STATUS RESTARTS AGE
pod/nginx-65899c769f-m4r2d 1/1 Running 1 23h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 23h
service/nginx NodePort 10.98.50.55 <none> 80:31746/TCP 23h
in host1 node curl host1:31746(nginx node port service)
root@host1:/home/vagrant# curl 10.0.0.159:31746
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
in host1 node curl host2:31746(nginx node port service)
root@host1:/home/vagrant# curl 10.0.0.167:31746
curl: (7) Failed to connect to 10.0.0.167 port 31746: Connection timed out
in host2 node curl host1:31746(nginx nodeport service)
root@host2:/home/vagrant# curl 10.0.0.159:31746
curl: (7) Failed to connect to 10.0.0.159 port 31746: Connection timed out
in host2 node curl host2:31746(nginx nodeport service)
root@host2:/home/vagrant# curl 10.0.0.167:31746
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
我使用的是您提供的Vagrantfile所建立的virtualbox，kubernetes version為底下所顯示的
root@host1:/home/vagrant# kubectl version
Client Version: version.Info{Major:"1", Minor:"10", GitVersion:"v1.10.4", GitCommit:"5ca598b4ba5abb89bb773071ce452e33fb66339d", GitTreeState:"clean", BuildDate:"2018-06-06T08:13:03Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"10", GitVersion:"v1.10.4", GitCommit:"5ca598b4ba5abb89bb773071ce452e33fb66339d", GitTreeState:"clean", BuildDate:"2018-06-06T08:00:59Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
@John-Lin 佈署的yaml如下：
apiVersion: apps/v1
kind: Deployment
metadata:
name: onos-deployment
labels:
app: onos
spec:
replicas: 1
selector:
matchLabels:
app: onos
template:
metadata:
labels:
app: onos
spec:
containers:
- name: onos
image: onosproject/onos
ports:
- containerPort: 8181
name: onos-gui
- containerPort: 6653
name: onos-openflow
- containerPort: 8180
name: onos-ctl
- containerPort: 6640
name: onos-ovsdb
- containerPort: 9876
name: onos-cluster
---
kind: Service
apiVersion: v1
metadata:
name: onos-service
spec:
selector:
app: onos
type: NodePort
ports:
- name: onos-gui
protocol: TCP
port: 8181
- name: onos-openflow
protocol: TCP
port: 6653
- name: onos-ctl
protocol: TCP
port: 8180
- name: onos-ovsdb
protocol: TCP
port: 6640
- name: onos-cluster
protocol: TCP
port: 9876
kubectl get node
NAME STATUS ROLES AGE VERSION
master Ready master 7d v1.10.3
node1 Ready <none> 7d v1.10.3
kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
onos-service NodePort 10.97.233.142 <none> 8181:32454/TCP,6653:30487/TCP,8180:30605/TCP,6640:30328/TCP,9876:31075/TCP 1h
kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE
onos-deployment-68967cd765-9mqzc 1/1 Running 0 1h 10.244.2.25 node1
當我在master node curl
在 master 直接存取 node:NodePort 或在node 直接存取 master:NodePort 是可行的嗎？我在一般的CNI環境測試也是不會通的
-
先確認vm裡的 master跟 node 的 host 在可以互相通訊 (master ping node and vice versa)
-
再開一個 VM host3 與 k8s 集群的物理節點同網域，使用node port 存取master:NodePort/node:NodePort (curl host2:31746 or host1:31746 ) 應該要可以通
@John-Lin 我這邊另外開一個kubernetes 環境，環境為兩台實體機器 一台為master 另外一台為node ，ip各為 10.0.0.227 ,10.0.0.228 cni 的部分採用calico ，部署方式採用kubeadm
root@k8s-B1:/home/ubuntu# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-b1 Ready master 1d v1.10.4
k8s-b2 Ready <none> 1d v1.10.4
root@k8s-B1:/home/ubuntu# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
calico-node-rvs2x 2/2 Running 0 1d
calico-node-s2pt7 2/2 Running 0 1d
etcd-sdn-k8s-b1 1/1 Running 0 1d
kube-apiserver-sdn-k8s-b1 1/1 Running 0 1d
kube-controller-manager-sdn-k8s-b1 1/1 Running 0 1d
kube-dns-86f4d74b45-kdzf8 3/3 Running 0 1d
kube-proxy-b8zg8 1/1 Running 0 1d
kube-proxy-h4nn2 1/1 Running 0 1d
kube-scheduler-sdn-k8s-b1 1/1 Running 0 1d
這邊我run 了nginx 的deployment 並且expose service。
root@k8s-B1:/home/ubuntu# kubectl run nginx --image nginx
root@k8s-B1:/home/ubuntu# kubectl expose deploy nginx --type NodePort --port 80
確認一下pod以及service的部分。
root@sdn-k8s-B1:/home/ubuntu# kubectl get pod,service -o wide
NAME READY STATUS RESTARTS AGE IP NODE
pod/nginx-65899c769f-78pvp 1/1 Running 0 8m 192.168.1.6 k8s-b2
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 1d <none>
service/nginx NodePort 10.111.112.254 <none> 80:32175/TCP 7m run=nginx
在master node上面去curl master node 的nginx service node port
root@k8s-B1:/home/ubuntu# curl 10.0.0.227:32175
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
在master node上去curl node1 的nginx service node port
root@k8s-B1:/home/ubuntu# curl 10.0.0.228:32175
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
@John-Lin
æç¨å¦å¤çç°å¢ï¼cni æ¯ç¨flannelå測試
åå¥çºmaster以ånode1ï¼ä¸æ¨£ä»¥ä¸é¢çyamlåä½ç½²
apiVersion: apps/v1
kind: Deployment
metadata:
name: onos-deployment
labels:
app: onos
spec:
replicas: 1
selector:
matchLabels:
app: onos
template:
metadata:
labels:
app: onos
spec:
containers:
- name: onos
image: onosproject/onos
ports:
- containerPort: 8181
name: onos-gui
- containerPort: 6653
name: onos-openflow
- containerPort: 8180
name: onos-ctl
- containerPort: 6640
name: onos-ovsdb
- containerPort: 9876
name: onos-cluster
---
kind: Service
apiVersion: v1
metadata:
name: onos-service
spec:
selector:
app: onos
type: NodePort
ports:
- name: onos-gui
protocol: TCP
port: 8181
- name: onos-openflow
protocol: TCP
port: 6653
- name: onos-ctl
protocol: TCP
port: 8180
- name: onos-ovsdb
protocol: TCP
port: 6640
- name: onos-cluster
protocol: TCP
port: 9876
å¨master 主æ©ä¸ curl
html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 Not Found</title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /. Reason:
<pre> Not Found</pre></p>
<hr /><i><small>Powered by Jetty:// 8.1.19.v20160209</small></i>
</body>
</html>